alma_course_loader 0.9.1

data/lib/alma_course_loader/filter.rb

@@ -0,0 +1,272 @@
+require 'json'
+require 'set'
+
+module AlmaCourseLoader
+  # Methods for parsing filter specification strings
+  # The primary method is parse_filter(str, extractors)
+  module FilterParser
+    # The regular expression describing a filter string: [!][extractor[op]]value
+    FILTER = Regexp.new('\s*(?<negate>!)?\s*' \
+                        '(' \
+                        '(?<extractor>[^\s]*)\s+' \
+                        '(?<method><=?|>=?|==|!=|~|=~|!~|keyin|valuein|in)\s+' \
+                        ')?' \
+                        '(?<value>.*)').freeze
+
+    # Filter operators mapped to method names
+    #
+    # The operators in filter expressions are converted to methods called on
+    # the filter value. Note that this means that the < and > operators are
+    # inverted ('field < filter-value' is equivalent to 'filter-value.>(field)')
+    # Operators not specified explicitly in FILTER_MAP are called directly as
+    # methods.
+    #
+    # The hash value is either:
+    # - a Symbol representing the method name to call on the filter value;
+    # - an array [method-name-symbol, negated] containing the method name symbol
+    #   and a Boolean indicating whether the method is implicitly negated,
+    #   e.g. !~ is !match(); the default is no implicit negation.
+    #
+    FILTER_MAP = {
+      :~ => :match,
+      :=~ => :match,
+      :!~ => [:match, true],
+      :< => :>,
+      :<= => :>=,
+      :>= => :<=,
+      :> => :<,
+      :in => :include?,
+      :keyin => :key?,
+      :valuein => :value?
+    }.freeze
+
+    # Parses a filter string and returns the parsed filter string components
+    # @param str [String] the filter string: [extractor][+|-]value
+    #   where value is a JSON string specifying the value(s) to match,
+    #         extractor is the optional name of an entry in the extractors hash,
+    # @param extractors [Hash<Symbol, Proc|Method>] a hash of named field
+    #   extractors referenced by the filter string; this may include a nil
+    #   key which specifies the default extractor if not given in the filter
+    #   string
+    # @return [Array] the parsed filter string components:
+    #   values, method, extractor, negate: [Object, Symbol, Proc, Boolean]
+    # @raise [ArgumentError] if the filter string is invalid
+    def parse_filter(str = nil, extractors = nil)
+      # Parse the filter string
+      raise ArgumentError, 'expected filter' if str.nil? || str.empty?
+      match = FILTER.match(str)
+      raise ArgumentError, "invalid filter: #{str}" if match.nil?
+      # Get the filter value(s)
+      value = Private.parse_value(match)
+      # Get the method used to compare the filter value(s)
+      method, method_negate = parse_method(match, value)
+      # Get the effective negate flag
+      negate = Private.negate_flag(match[:negate] == '!', method_negate)
+      # Get the named extractor from the hash or block
+      extractor = Private.parse_extractor(match, extractors)
+      # Return the parsed filter string components
+      [value, method, extractor, negate]
+    end
+
+    # Validates and returns the method symbol and negation flag
+    # @param match [MatchData] the parsed filter string
+    # @param values [Object] the filter value(s)
+    # @return [Array] the method symbol and negate flag
+    # @raise [ArgumentError] if the specified method is not supported by the
+    #   filter values
+    def parse_method(match, values)
+      method_info = Private.method_symbol(match, values)
+      method = method_info[0]
+      unless values.respond_to?(method)
+        raise ArgumentError, "invalid method: #{values.class.name}\##{method}"
+      end
+      method_info
+    end
+
+    # Private helpers
+    module Private
+      # Returns true if the value parameter is a collection
+      # @param value [Object] the value to test
+      # @return [Boolean] true if the value is a collection, false otherwise
+      def self.collection?(value)
+        value.is_a?(Array) || value.is_a?(Hash) || value.is_a?(Set)
+      end
+
+      # Returns a Proc implementing the filter match
+      # @param values [Object] the filter values
+      # @param method [Symbol] the method to call against the filter values
+      # @param negate [Boolean] if true, negate the result of the method
+      # @return [Proc] the Proc implementing the filter
+      def self.matcher(values, method = nil, negate = false)
+        method, negate = method_symbol(method, values) if method.nil?
+        proc do |value|
+          result = values.send(method, value)
+          negate ? !result : result
+        end
+      end
+
+      # Returns the method symbol or appropriate default if not specified
+      # If no method is specified in the filter string, the default is :include?
+      # for Arrays, Hashes and Sets, :match for regular expressions and :== for
+      # everything else.
+      # @param match [MatchData] the parsed filter string
+      # @param values [Object] the filter value(s)
+      # @return [Array] the method symbol and negation flag (false|true)
+      def self.method_symbol(match, values)
+        method = match[:method]
+        # Return the default if no method is specified
+        return method_symbol_default(values) if method.nil? || method.empty?
+        method = method.to_sym
+        # Return the method as specified unless it's mapped
+        return method, false unless FILTER_MAP.key?(method)
+        # Otherwise return the mapped filter
+        method, negate = FILTER_MAP[method]
+        [method, negate.nil? ? false : negate]
+      end
+
+      # Returns a default method symbol based on the filter value type
+      # @param values [Object] the filter value(s)
+      # @return [Array] the default method symbol and negation flag (false|true)
+      def self.method_symbol_default(values)
+        return :include?, false if collection?(values)
+        return :match, false if values.is_a?(Regexp)
+        [:==, false]
+      end
+
+      # Returns the actual filter negate flag based on the requested filter
+      # negate flag and the compare method's negate flag
+      # @param filter_negate [Boolean] the requested filter negate flag
+      # @param method_negate [Boolean] the compare method's negate flag
+      # @return [Boolean] the effective negate flag
+      def self.negate_flag(filter_negate = false, method_negate = false)
+        # The effective negate flag is true only if exactly one of the arguments
+        # is true (filter_flag xor method_negate) since double negation cancels
+        filter_negate != method_negate
+      end
+
+      # Returns the extractor Proc specified by a parsed filter string
+      # @param match [MatchData] the parsed filter string
+      # @param extractors [Hash<Symbol, Proc|Method>] the extractors
+      # @raise [ArgumentError] if a specified extractor is invalid or there is
+      #   no default extractor in extractors when required
+      def self.parse_extractor(match, extractors = nil)
+        raise ArgumentError, 'extractors required' \
+          if extractors.nil? || extractors.empty?
+        e = match[:extractor]
+        extractor = extractors[e ? e.to_sym : nil]
+        return extractor unless extractor.nil?
+        raise ArgumentError, "invalid extractor: #{e}" if e
+        raise ArgumentError, 'no default extractor'
+      end
+
+      # Returns a parsed regular expression
+      # @param regexp [String] the regular expression from the value string
+      # @return [Regexp] the parsed regular expression
+      # @raise [ArgumentError] if the regular expression cannot be parsed
+      def self.parse_regexp(regexp)
+        # Remove enclosing / if present
+        first = regexp.start_with?('/') ? 1 : 0
+        last = regexp.end_with?('/') ? -2 : -1
+        Regexp.new(regexp[first..last])
+      rescue RegexpError
+        raise ArgumentError, "invalid regular expression: #{regexp}"
+      end
+
+      # Returns the parsed value string
+      # @param match [MatchData] the parsed filter string
+      # @return [Object] the filter value(s)
+      # @raise [ArgumentError] if the value string cannot be parsed
+      def self.parse_value(match)
+        value = match[:value]
+        return nil if value.nil?
+        # Parse strings starting with / as regular expressions
+        return parse_regexp(value) if value.start_with?('/')
+        # Parse all other strings as JSON
+        JSON.parse(value)
+      rescue JSON::ParserError
+        raise ArgumentError, "invalid value: #{value}"
+      end
+    end
+  end
+
+  # Implements a course filter
+  class Filter
+    extend FilterParser
+
+    # @!attribute [rw] extractor
+    #   @return [Proc, Method] a block accepting a year, course and cohort and
+    #     returning the value used in filter matching
+    attr_accessor :extractor
+
+    # @!attribute [rw] method
+    #   @return [Symbol] the method to call against the values
+    attr_accessor :method
+
+    # @!attribute [rw] negate
+    #   @return [Boolean] if true, the filter result is negated
+    attr_accessor :negate
+
+    # @!attribute [rw] values
+    #   @return [Object] the value or collection to match against
+    attr_accessor :values
+
+    # Parses a filter string and returns a Filter instance
+    # @param str [String] the filter string: [extractor][+|-]value
+    #   where value is a JSON string specifying the value(s) to match,
+    #         extractor is the optional name of an entry in the extractors hash,
+    # @param extractors [Hash<Symbol, Proc|Method>] a hash of named field
+    #   extractors referenced by the filter string; this may include a nil
+    #   key which specifies the default extractor if not given in the filter
+    #   string
+    # @raise [ArgumentError] if the filter string is invalid
+    def self.parse(str = nil, extractors = nil)
+      new(*parse_filter(str, extractors))
+    end
+
+    # Initialises a new Filter instance
+    # @param values [Object] the value or collection to match against
+    # @param method [Symbol] the method symbol applied to the filter values
+    # @param extractor [Proc, Method] a block accepting a year, course and
+    #   cohort and returning the value used in filter matching.
+    #   This may be also passed as a code block.
+    # @param negate [Boolean] if true, negate the result of the filter
+    # @raise [ArgumentError]
+    # @return [void]
+    def initialize(values = nil, method = nil, extractor = nil, negate = false,
+                   &block)
+      self.method = method
+      self.negate = negate
+      self.extractor = extractor || block
+      self.values = values
+      @matcher = matcher_proc(method)
+    end
+
+    # Applies the filter to a course/cohort
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the course cohort
+    # @return [Boolean] true if the course passes the filter, false otherwise
+    def call(year = nil, course = nil, cohort = nil)
+      # Get the value for matching
+      value = extractor.call(year, course, cohort)
+      # Test against the filter values (true => match, false => no match)
+      result = @matcher.call(value)
+      # Return the result with appropriate negation
+      negate ? !result : result
+    end
+
+    private
+
+    # Returns a Proc instance which calls the specified method on the filter
+    # value object
+    # @param method [Method, Proc, Symbol] the method (Method/Proc arguments are
+    #   returned as-is)
+    # @return [Proc] the Proc instance
+    def matcher_proc(method)
+      # Return Method/Proc arguments unchanged
+      return method if method.is_a?(Method) || method.is_a?(Proc)
+      # Otherwise return a Proc which invokes method on the filter values
+      proc { |value| values.send(method, value) }
+    end
+  end
+end

data/lib/alma_course_loader/reader.rb

@@ -0,0 +1,298 @@
+require 'alma_course_loader/version'
+
+module AlmaCourseLoader
+  # The abstract base class for course readers.
+  #
+  # A course reader is responsible for reading course data from a data source
+  #   and iterating over each course element. A course element is either the
+  #   course itself or, if the course is divided into cohorts, a single cohort
+  #   of the course.
+  #
+  # It is expected that courses are retrieved by year, and the iterators accept
+  #   a list of years for which courses are required. The implementation and
+  #   type of years, courses and cohorts is deferred to the subclasses.
+  #
+  # Filters may be defined which determine whether a single course element is
+  #   processed or ignored. The course element must pass all filters before it
+  #   is processed.
+  #
+  # The years and filters controlling course element iteration can be passed
+  #   to the constructor, which defines defaults for subsequent iterators, or to
+  #   the #each method, which defines the criteria for that iteration. When
+  #   creating a reader which will be passed to a writer, always define the
+  #   iterator criteria in the reader constructor, as the writer uses the
+  #   default iterator.
+  #
+  # There are two classes of iterators. The first invokes its block with the
+  #   course details: yield(year, course, cohort, instructors)
+  # The second, used by the Writer class, invokes its block with an array of
+  #   data suitable for writing as a row in a course loader CSV file: yield(row)
+  #
+  # @abstract Subclasses must implement the following methods:
+  #   #courses
+  #     returns a list of course objects
+  #   #course_cohorts(course)
+  #     returns a list of cohorts for the course or nil if cohorts are not used
+  #   #current_academic_year
+  #     returns the current academic year
+  #   #instructors(year, course, cohort)
+  #     returns a list of instructors for the course
+  #   #row_data(array, year, course, cohort, instructors)
+  #     configures an array of formatted course information suitable for writing
+  #     to an Alma course loader CSV file
+  #
+  # @example Create course filters
+  #   # Define extractors to retrieve values from a course element for matching
+  #   # Extractors are called with the year, course and cohort arguments
+  #   get_code = proc { |year, course, cohort| course.code }
+  #   get_title = proc { |year, course, cohort| course.title }
+  #
+  #   # Define a list of course codes and a regular expression matching titles
+  #   codes = ['COMPSCI101', 'PHYSICS101', 'MAGIC101']
+  #   titles = Regexp.compile('dissertation|(extended|ma) essay|test', true)
+  #
+  #   # Define a filter which passes only the specified course codes
+  #   include_codes = AlmaCourseLoader::Filter.new(codes, :include, get_code)
+  #
+  #   # Define a filter which passes all codes except those specified
+  #   exclude_codes = AlmaCourseLoader::Filter.new(codes, :exclude, get_code)
+  #
+  #   # Define a filter which passes all titles matching the regular expression
+  #   include_titles = AlmaCourseLoader::Filter.new(titles, :include, get_title)
+  #
+  #   # Define a filter which passes all titles except those matching the regexp
+  #   exclude_titles = AlmaCourseLoader::Filter.new(titles, :exclude, get_title)
+  #
+  # @example Create a reader with default selection criteria
+  #   # Filters are passed in an array, course elements must pass all filters
+  #   reader = Reader.new(2016, 2017, filters: [exclude_codes, include_titles])
+  #
+  # @example Iterate course elements using default selection criteria
+  #   reader.each { |year, course, cohort, instructors| ... }
+  #
+  # @example Iterate course elements using specific selection criteria
+  #   # Use an empty array to override filters. filters: nil will use the
+  #   # default filters.
+  #   reader.each(2012, filters: []) { |year, course, cohort, instructors| ... }
+  #
+  # @example Iterate course CSV rows
+  #   reader.each_row { |row_array| ... }
+  #   reader.each_row(2012, filters: []) { |row_array| ... }
+  #
+  class Reader
+    # Initialises a new Reader instance
+    # Positional parameters are the default years to iterate over
+    # @param current_year [Object] the current academic year
+    # @param filters [Array<AlmaCourseLoader::Filter>] default course filters
+    def initialize(*years, current_year: nil, filters: nil)
+      @current_academic_year = current_year || current_academic_year
+      @filters = filters
+      @years = years.nil? || years.empty? ? [@current_academic_year] : years
+    end
+
+    # Iterate over the courses for specified years. Only courses which pass
+    # the filters are passed to the block.
+    # Positional parameters are years
+    # @param filters [Array<AlmaCourseLoader::Filter>] course filters
+    # @return [void]
+    # @yield [year, course, cohort, year, instructors] Passes the course to the
+    #   block
+    # @yieldparam year [Object] the course year
+    # @yieldparam course [Object] the course
+    # @yieldparam cohort [Object] the course cohort
+    # @yieldparam instructors [Array<Object>] the course instructors
+    def each(*years, filters: nil, &block)
+      # Process courses for each year
+      years = @years if years.nil? || years.empty?
+      years.each { |year| each_course_in_year(year, filters: filters, &block) }
+      nil
+    end
+
+    # Iterates over the cohorts in a course, or just the course itself if
+    # cohort processing is disabled. Only courses/cohorts which pass the filters
+    # are passed to the block.
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param filters [Array<AlmaCourseLoader::Filter>] course filters
+    # @return [void]
+    # @yield [year, course, cohort, year, instructors] Passes the course to the
+    #   block
+    # @yieldparam year [Object] the course year
+    # @yieldparam course [Object] the course
+    # @yieldparam cohort [Object] the course cohort
+    # @yieldparam instructors [Array<Object>] the course instructors
+    def each_cohort_in_course(year, course, filters: nil, &block)
+      cohorts = course_cohorts(year, course)
+      if cohorts.nil?
+        # Process the course
+        process_course(year, course, nil, filters, &block)
+      else
+        cohorts.each do |cohort|
+          # Process each cohort of the course
+          process_course(year, course, cohort, filters, &block)
+        end
+      end
+    end
+
+    # Iterates over the cohorts in a course, or just the course itself if
+    # cohort processing is disabled. Only courses/cohorts which pass the filters
+    # are passed to the block.
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param filters [Array<AlmaCourseLoader::Filter>] course filters
+    # @return [void]
+    # @yield [row] Passes the course to the block
+    # @yieldparam row [Array<String>] the course as a CSV row (array)
+    def each_cohort_in_course_row(year, course, filters: nil, &block)
+      each_cohort_in_course(year, course, filters: filters) do |*args|
+        row(*args, &block)
+      end
+    end
+
+    # Iterate over the courses for the specified year. Only courses which pass
+    # the filters are passed to the block.
+    # @param year [Object] the year
+    # @param filters [Array<AlmaCourseLoader::Filter>] filters selecting courses
+    # @return [void]
+    # @yield [year, course, cohort, year, instructors] Passes the course to the
+    #   block
+    # @yieldparam year [Object] the course year
+    # @yieldparam course [Object] the course
+    # @yieldparam cohort [Object] the course cohort
+    # @yieldparam instructors [Array<Object>] the course instructors
+    def each_course_in_year(year, filters: nil, &block)
+      # Simplify the test for filter existence
+      filters ||= @filters
+      filters = nil if filters.is_a?(Array) && filters.empty?
+      # Get all courses for the year
+      courses(year).each do |course|
+        each_cohort_in_course(year, course, filters: filters, &block)
+      end
+      nil
+    end
+
+    # Iterate over the courses for the specified year. Only courses which pass
+    # the filters are passed to the block.
+    # @param year [Object] the year
+    # @param filters [Array<AlmaCourseLoader::Filter>] course filters
+    # @return [void]
+    # @yield [row] Passes the course to the block
+    # @yieldparam row [Array<String>] the course as a CSV row (array)
+    def each_course_in_year_row(year, filters: nil, &block)
+      each_course_in_year(year, filters: filters) { |*args| row(*args, &block) }
+    end
+
+    # Iterate over the courses for specified years. Only courses which pass
+    # the filters are passed to the block.
+    # Positional parameters are years
+    # @param filters [Array<AlmaCourseLoader::Filter>] course filters
+    # @return [void]
+    # @yield [row] Passes the course to the block
+    # @yieldparam row [Array<String>] the course as a CSV row (array)
+    def each_row(*years, filters: nil, &block)
+      each(*years, filters: filters) { |*args| row(*args, &block) }
+    end
+
+    protected
+
+    # Returns a list of courses for the specified year
+    # @abstract Subclasses must implement this method.
+    # @param year [Object] the course year
+    # @return [Array<Object>] the courses for the year
+    def courses(year)
+      []
+    end
+
+    # Returns a list of cohorts for the specified course
+    # @abstract Subclasses should implement this method. If courses are not
+    #   divided into cohorts, this method must return nil.
+    # @param course [Object] the course
+    # @return [Array<Object>, nil] the cohorts for the course, or nil to disable
+    #   cohort processing
+    def course_cohorts(year, course)
+      nil
+    end
+
+    # Returns the current academic year
+    # @abstract Subclasses should implement this method
+    # @return [Object] the current academic year
+    def current_academic_year
+      nil
+    end
+
+    # Applies filters to the course/cohort to determine whether to process
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the course cohort
+    # @return [Boolean] true if the course is to be processed, false if not
+    def filter(year, course, cohort, filters)
+      # Return true if no filters are specified
+      return true if filters.nil? || filters.empty?
+      # Return false if any of the filters returns false
+      filters.each { |f| return false unless f.call(year, course, cohort) }
+      # At this point all filters returned true
+      true
+    end
+
+    # Returns a list of instructors for the course/cohort
+    # @abstract Subclasses should implement this method
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the course cohort
+    # @return [Array<Object>] the course instructors
+    def instructors(year, course, cohort)
+      []
+    end
+
+    # Filters and processes a course
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the LUSI course cohort
+    # @param filters [Array<AlmaCourseLoader::Filter>] the course filters
+    # @return [Boolean] true if the course was processed, false otherwise
+    # @yield Passes the course to the block
+    # @yieldparam year [Object] the course year
+    # @yieldparam course [Object] the course
+    # @yieldparam cohort [Object] the course cohort
+    # @yieldparam instructors [Array<Object>] the course instructors
+    def process_course(year, course, cohort = nil, filters = nil)
+      # The course must pass all filters
+      return false unless filter(year, course, cohort, filters)
+      # Get the course instructors
+      course_instructors = instructors(year, course, cohort)
+      # Pass the details to the block
+      yield(year, course, cohort, course_instructors) if block_given?
+      # Indicate that the course was processed
+      true
+    end
+
+    # Returns a CSV row (array) for a specific course/cohort
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the course cohort
+    # @param instructors [Array<Object>] the
+    #   course instructor enrolments
+    # @yield Passes the CSV row array to the block
+    # @yieldparam row [Array<String>] the CSV row array
+    def row(year = nil, course = nil, cohort = nil, instructors = nil)
+      # Create and populate the CSV row
+      data = Array.new(31)
+      row_data(data, year, course, cohort, instructors)
+      # Pass the row to the block
+      yield(data) if block_given?
+      # Return the row
+      data
+    end
+
+    # Populates the CSV row (array) for a specific course/cohort
+    # @abstract Subclasses must implement this method
+    # @param data [Array<String>] the CSV row (array)
+    # @param year [Object] the course year
+    # @param course [Object] the course
+    # @param cohort [Object] the course cohort
+    # @param instructors [Array<Object>] the course instructors
+    def row_data(data, year, course, cohort, instructors)
+      data
+    end
+  end
+end