aspire 0.1.0

data/lib/aspire/caching/cache_logger.rb

@@ -0,0 +1,63 @@
+require 'logger'
+
+require 'aspire/exceptions'
+
+module Aspire
+  # Tools for building a caching from the Aspire APIs
+  module Caching
+    # A wrapper class for Logger adding utility methods
+    class CacheLogger
+      # @!attribute [rw] logger
+      #   @return [Logger] the logger
+      attr_accessor :logger
+
+      # Delegates missing methods to the logger
+      # @param method [Symbol] the method name
+      # @param args [Array] the method arguments
+      # @param block [Proc] the method code block
+      # @return [Object] the method result
+      def method_missing(method, *args, &block)
+        # Do not fail if logger is undefined
+        return nil unless logger
+        # Fail if logger does not respond to this method
+        super unless logger.respond_to?(method)
+        # Delegate to the logger method
+        logger.public_send(method, *args, &block)
+      end
+
+      # Delegates missing method respond_to? to the wrapped logger
+      # @param method [Symbol] the method name
+      # @return [Boolean] true if the wrapped logger responds to the method
+      def respond_to_missing?(method)
+        # If logger is undefined, all missing methods are accepted
+        logger ? logger.respond_to?(method) : true
+      end
+
+      # Initialises a new CacheLogger instance
+      # @param logger [Logger] the logger
+      def initialize(logger = nil)
+        self.logger = logger
+      end
+
+      # Logs and raises an exception
+      # @param message [String] the error message
+      # @param exception [Class] the class of the exception to be raised
+      # @param level [Symbol] the logger level (default: Logger::ERROR)
+      # @raise [Aspire::Caching::Exceptions::Error]
+      def log_exception(message, exception = nil, level: nil)
+        log(level || Logger::ERROR, message)
+        raise exception || Aspire::Exceptions::Error, message
+      end
+
+      # Logs an event and returns its first argument
+      # - allows for compact code such as 'return log_return(result, msg,...)'
+      # @param result [Object] the return value of the method
+      # @param (see #log)
+      # @return [Object] the result argument
+      def log_return(result, *args, **kwargs)
+        log(*args, **kwargs)
+        result
+      end
+    end
+  end
+end

data/lib/aspire/caching/util.rb

@@ -0,0 +1,210 @@
+require 'fileutils'
+require 'json'
+require 'uri'
+
+require 'aspire/enumerator/linked_data_uri_enumerator'
+require 'aspire/exceptions'
+require 'aspire/util'
+
+module Aspire
+  # Tools for building a caching from the Aspire APIs
+  module Caching
+    # Cache utility methods
+    module Util
+      include Aspire::Util
+
+      # Rules for determining whether an object URL is cacheable
+      # Each rule is a Proc which accepts a parsed URL from #parse_url and the
+      # CacheEntry instance, and returns true if the object is cacheable or
+      # false if not. Rules are applied in the order specified and all rules
+      # must return true for an object to be cacheable.
+      CACHEABLE = [
+        # The URL must be set and the host must mach the canonical tenancy host
+        proc { |u, e| u && u[:tenancy_host] == e.cache.tenancy_host },
+        # Catalog objects are not cacheable
+        proc { |u, _e| u[:type] != 'catalog' },
+        # User objects themselves are not cacheable but child objects e.g. notes
+        # are cacheable
+        proc { |u, _e| u[:type] != 'users' || !u[:child_type].nil? },
+        # Importance URI values are not cacheable
+        proc do |u, _e|
+          u[:type] != 'config' || !u[:id].to_s.start_with?('importance')
+        end
+      ].freeze
+
+      # Adds a prefix to a filename
+      # @param filename [String] the filename
+      # @param prefix [String] the prefix
+      # @return [String] the filename with prefix
+      def add_filename_prefix(filename, prefix)
+        filename = filename.rpartition(File.basename(filename))
+        filename[1] = "#{prefix}#{filename[1]}"
+        filename.join
+      end
+
+      # Adds a suffix to a filename preserving any file extension
+      # e.g. add_filename_suffix('file.txt', '-suffix') == 'file-suffix.txt'
+      # @param filename [String] the filename
+      # @param suffix [String] the suffix
+      # @return [String] the filename with suffix
+      def add_filename_suffix(filename, suffix)
+        f = filename.split(File::SEPARATOR)
+        # If the filename is '.' or '..' add the suffix to the parent path,
+        # otherwise add it to the basename
+        i = %w[. ..].include?(f[-1]) ? -2 : -1
+        # Split the basename around the file extension and prepend the suffix
+        # to the extension
+        if f[i]
+          file_ext = f[i].rpartition(File.extname(f[i]))
+          file_ext[1] = "#{suffix}#{file_ext[1]}"
+          f[i] = file_ext.join
+        end
+        # Reconstruct the filename, preserving any trailing path separator
+        f.push('') if filename.end_with?(File::SEPARATOR)
+        File.join(f)
+      end
+
+      # Parses the URL and checks that it is cacheable
+      # @param u [String] the URL of the API object
+      # @return [MarchData] the parsed URL
+      # @raise [Aspire::Exceptions::NotCacheable] if the URL is not
+      #   cacheable
+      def cacheable_url(u)
+        # All rules must return true for the URL to be cacheable
+        u = parse_url(u)
+        CACHEABLE.each do |r|
+          raise Aspire::Exceptions::NotCacheable unless r.call(u, self)
+        end
+        # Return the parsed URL
+        u
+      end
+
+      # Returns true if the directory path has no more parents, false otherwise
+      # @param dir [String] the directory path
+      # @param root [String] the directory root - paths above this are ignored
+      # @return [Boolean] true if there are no more parents, false otherwise
+      def end_of_path?(dir, root = nil)
+        dir.nil? || dir.empty? || dir == '.' || dir == root
+      end
+
+      # Creates a directory and its parents, logs errors
+      # @param dir [String] the directory name
+      # @param logger [Aspire::Caching::CacheLogger] the logger for messages
+      # @param failure [String] the error message on failure
+      # @return [void]
+      # @raise [ArgumentError] if the directory is not specified
+      # @raise [Aspire::Cache::Exceptions::WriteError] if the operation fails
+      def mkdir(dir, logger = nil, success = nil, failure = nil)
+        raise ArgumentError, 'Directory expected' if dir.nil? || dir.empty?
+        FileUtils.mkdir_p(dir, mode: mode)
+        return if logger.nil? || success.nil? || success.empty?
+        logger.log(Logger::DEBUG, success)
+      rescue SystemCallError => e
+        failure ||= "Create directory #{dir} failed"
+        message = "#{failure}: #{e}"
+        raise WriteError, message if logger.nil?
+        logger.log_exception(message, WriteError)
+      end
+
+      # Returns the list of URI references from a linked data API object
+      # @param url [String] the URL of the API object
+      # @param data [Hash] the parsed JSON data for the object
+      # @return [Array<String>] the list of URIs referenced by the object
+      def references(url, data = nil)
+        return [] if data.nil? || data.empty?
+        # Enumerate the URIs and add them as keys of a hash to de-duplicate
+        enum = Aspire::Enumerator::LinkedDataURIEnumerator.new.enumerator(url, data)
+        uris = {}
+        enum.each { |_k, hash, _i| uris[hash['value']] = true }
+        # Return the list of URIs
+        uris.keys
+      end
+
+      # Removes the specified files
+      # @param glob [String] the file pattern to be removed
+      # @param logger [Aspire::Caching::CacheLogger] the logger for messages
+      # @param success [String] the text for success log messages
+      # @param failure [String] the text for failure exception/log messages
+      # @return [void]
+      # @raise [Aspire::Cache::Exceptions::RemoveError] if the removal fails
+      def rm(glob, logger = nil, success = nil, failure = nil)
+        raise ArgumentError, 'file path required' if glob.nil? || glob.empty?
+        FileUtils.rm_rf(Dir.glob(glob), secure: true)
+        return if logger.nil? || success.nil? || success.empty?
+        logger.log(Logger::INFO, success)
+      rescue SystemCallError => e
+        failure ||= "Remove #{glob} failed"
+        message = "#{failure}: #{e}"
+        raise RemoveError, message if logger.nil?
+        logger.log_exception("#{failure}: #{e}", RemoveError)
+      end
+
+      # Remove empty directories in a directory path
+      # @param path [String] the starting file or directory
+      # @param root
+      # @return [void]
+      # @raise [Aspire::Exceptions::RemoveError] if the operation fails
+      def rmdir_empty(path, root)
+        # The starting path is assumed to be a filename, so we append a dummy
+        # filename if it's a directory
+        path = File.directory?(path) ? File.join(path, '.') : path
+        loop do
+          # Get the parent of the current directory/file
+          path = File.dirname(path)
+          # Stop at the end of the directory path or a non-empty directory
+          break if end_of_path?(path, root) || !Dir.empty?(path)
+          # Remove the directory
+          Dir.rmdir(path)
+        end
+      rescue Errno::ENOTEMPTY, Errno::ENOTDIR
+        # Stop without error if the directory is not empty or not a directory
+        nil
+      rescue SystemCallError => e
+        raise RemoveError, "Rmdir #{dir} failed: #{e}"
+      end
+
+      # Removes the file extension from a path
+      # @param path [String] the file path
+      # @return [String] the file path with any extension removed
+      def strip_ext(path)
+        path.rpartition(File.extname(path))[0]
+      end
+
+      # Removes a prefix from a filename
+      # @param filename [String] the filename
+      # @param prefix [String] the prefix
+      # @return [String] the filename without prefix
+      def strip_filename_prefix(filename, prefix)
+        f = filename.rpartition(File.basename(filename))
+        f[1] = strip_prefix(f[1], prefix)
+        f.join
+      end
+
+      # Removes a suffix from a filename
+      # @param filename [String] the filename
+      # @param suffix [String] the suffix
+      # @return [String] the filename without suffix
+      def strip_filename_suffix(filename, suffix)
+        f = filename.rpartition(File.extname(filename))
+        f[0] = strip_suffix(f[0], suffix)
+        f.join
+      end
+
+      # Removes a prefix from a string
+      # @param str [String] the string to remove the prefix from
+      # @param prefix [String] the prefix to remove
+      # @return [String] the string with the prefix removed
+      def strip_prefix(str, prefix)
+        str.start_with?(prefix) ? str.slice(prefix.length..-1) : str
+      end
+
+      # Removes a suffix from a string
+      # @param str [String] the string to remove the suffix from
+      # @param suffix [String] the suffix to remove
+      # @return [String] the string with the suffix removed
+      def strip_suffix(str, suffix)
+        str.end_with?(suffix) ? str.slice(0...-suffix.length) : str
+      end
+    end
+  end
+end

data/lib/aspire/cli/cache_builder.rb

@@ -0,0 +1,123 @@
+require 'aspire/cli/command'
+require 'aspire/enumerator/report_enumerator'
+require 'aspire/util'
+require 'logglier'
+require 'dotenv'
+
+module Aspire
+  module CLI
+
+    class CacheBuilder < Command
+
+      def execute
+
+        unless (env_file.nil? || env_file.empty?)
+          Dotenv.load(env_file)
+        end
+
+        @json_api = json_api
+        @linked_data_api = linked_data_api
+        @logger = create_logger log_to_file?
+        @cache_path = ENV['ASPIRE_CACHE_PATH']
+        @list_report = ENV['ASPIRE_LIST_REPORT']
+        @mode = ENV['ASPIRE_CACHE_MODE']
+        @mode = @mode.nil? || @mode.empty? ? 0o700 : @mode.to_i(8)
+        cache = Aspire::Caching::Cache.new(@linked_data_api, @json_api, @cache_path,
+                                           logger: @logger)
+        @builder = Aspire::Caching::Builder.new(cache)
+
+        if list_uri.nil? || list_uri.empty?
+
+          raise ArgumentError if privacy_control.nil? || privacy_control.empty?
+
+          puts "Caching all lists that match arguments"
+
+          lists = list_enumerator time_period_list, status, privacy_control
+
+          @builder.build(lists)
+
+          puts "Finished caching all lists that match arguments"
+
+        else
+          puts "Caching list #{list_uri}"
+          @builder.write_list(list_uri)
+          puts "Finished caching list"
+        end
+
+      end
+
+      private
+
+      def list_enumerator(time_periods=nil, status=nil, privacy_control=nil)
+
+        filters = []
+
+        if time_periods.nil? || time_periods.empty? || time_periods == ['']
+          time_periods = [nil, '']
+        end
+
+        filters.push(proc { |row| time_periods.include?(row['Time Period']) })
+
+        unless status.nil? || status.empty?
+          filters.push(proc { |row| row['Status'].to_s.start_with?(status) })
+        end
+
+        unless privacy_control.nil? || status.empty?
+          filters.push(proc { |row| row['Privacy Control'] == privacy_control })
+        end
+
+        Aspire::Enumerator::ReportEnumerator.new(@list_report, filters)
+            .enumerator
+      end
+
+      def json_api
+        @api_available = ENV['ASPIRE_API_AVAILABLE'] == 'true'
+        @api_client_id = ENV['ASPIRE_API_CLIENT_ID']
+        @api_secret = ENV['ASPIRE_API_SECRET']
+        @tenant = ENV['ASPIRE_TENANT']
+        Aspire::API::JSON.new(@api_client_id, @api_secret, @tenant,
+                              **api_opts)
+      end
+
+      def api_opts
+        @ssl_ca_file = ENV['SSL_CA_FILE']
+        @ssl_ca_path = ENV['SSL_CA_PATH']
+        {
+            ssl_ca_file: @ssl_ca_file,
+            ssl_ca_path: @ssl_ca_path
+        }
+      end
+
+      def linked_data_api
+        @api_available = ENV['ASPIRE_API_AVAILABLE'] == 'true'
+        @linked_data_root = ENV['ASPIRE_LINKED_DATA_ROOT']
+        @tenant = ENV['ASPIRE_TENANT']
+        @tenancy_host_aliases = ENV['ASPIRE_TENANCY_HOST_ALIASES'].to_s.split(';')
+        @tenancy_root = ENV['ASPIRE_TENANCY_ROOT']
+        Aspire::API::LinkedData.new(@tenant,
+                                    linked_data_root: @linked_data_root,
+                                    tenancy_host_aliases: @tenancy_host_aliases,
+                                    tenancy_root: @tenancy_root,
+                                    **api_opts)
+      end
+
+      def create_logger log_to_file
+
+        @log_file = ENV['ASPIRE_LOG']
+
+        if log_to_file
+          logger = Logger.new("| tee #{@log_file}") # @log_file || STDOUT)
+          logger.datetime_format = '%Y-%m-%d %H:%M:%S'
+          logger.formatter = proc do |severity, datetime, _program, msg|
+            "#{datetime} [#{severity}]: #{msg}\n"
+          end
+          return logger
+        end
+
+        Logglier.new("https://logs-01.loggly.com/inputs/#{ENV['LOGGLIER_TOKEN']}/tag/#{ENV['LOGGLIER_TAG']}/", :threaded => true, :format => :json)
+      end
+
+    end
+
+  end
+end

data/lib/aspire/cli/command.rb

@@ -0,0 +1,20 @@
+require 'clamp'
+
+module Aspire
+  module CLI
+
+    class Command < Clamp::Command
+
+      # option ['-c', '--hierarchy-code'], 'HIERARCHY_CODE', 'the hierarchy code (module etc.)'
+      option ['-e', '--env-file'], 'ENV_FILE', 'file containing env variable key value pairs'
+      option ['-l', '--list-uri'], 'LIST_URI', 'the list URI'
+      option ['-t', '--time-period'], 'TIME_PERIOD', 'the time period (2016-17 etc.)', :multivalued => true
+      option ['-p', '--privacy-control'], 'PRIVACY_CONTROL', 'the list privacy control (Public etc)'
+      option ['-s', '--status'], 'STATUS', 'the list status control (Published etc)'
+      option ['-c', '--clear-cache'], :flag, 'clear cache before running', default: false
+      option ['-f', '--log-to-file'], :flag, 'log output to file', default: false
+
+    end
+
+  end
+end

data/lib/aspire/enumerator/base.rb

@@ -0,0 +1,29 @@
+module Aspire
+  # Enumerator classes for Aspire reading list processing
+  module Enumerator
+    # The abstract base class for enumerator classes
+    # @abstract Subclasses must implement #enumerate accepting the parameters
+    #   passed to #enumerator and yielding values to self.yielder
+    class Base
+      # The Enumerator::Yielder instance from an Enumerator.new call
+      # @!attribute [rw] yielder
+      #   @return [Enumerator::Yielder] the yielder instance from an Enumerator
+      attr_accessor :yielder
+
+      # Enumerates the data passed in its arguments
+      # @abstract Subclasses must implement this method
+      def enumerate(*args, **kwargs)
+        raise NotImplementedError
+      end
+
+      # Returns an enumerator enumerating property/value pairs of JSON data
+      # @return [Enumerator] the enumerator
+      def enumerator(*args, **kwargs)
+        ::Enumerator.new do |yielder|
+          self.yielder = yielder
+          enumerate(*args, **kwargs)
+        end
+      end
+    end
+  end
+end