tax_generator 0.0.1

data/lib/tax_generator/classes/processor.rb

@@ -0,0 +1,270 @@
+require_relative '../helpers/application_helper'
+module TaxGenerator
+  # class used to process xml files and create html files
+  #
+  # @!attribute options
+  #   @return [Hash] the options that can determine the input and output files and folders
+  #
+  # @!attribute worker_supervisor
+  #   @return [Celluloid::SupervisionGroup] the supervision group that supervises workers
+  # @!attribute workers
+  #   @return [Celluloid::Actor] the actors that will work on the jobs
+  # @!attribute taxonomy
+  #   @return [TaxGenerator::TaxonomyTree] the taxonomy tree that holds the nodes from the taxonomy xml document
+  # @!attribute jobs
+  #   @return [Hash] each key from the job list is the job id, and the value is the job itself
+  # @!attribute job_to_worker
+  #   @return [Hash] each key from the list is the job id, and the value is the worker that will handle the job
+  # @!attribute worker_to_job
+  #   @return [Hash] each key from the list is the workers mailbox address, and the value is the job being handled by the worker
+  # @!attribute condition
+  #   @return [Celluloid::Condition] the supervision group that supervises workers
+  class Processor
+    include Celluloid
+    include Celluloid::Logger
+    include Celluloid::Notifications
+    include TaxGenerator::ApplicationHelper
+
+    attr_reader :options, :worker_supervisor, :workers, :taxonomy, :jobs, :job_to_worker, :worker_to_job, :condition
+
+    trap_exit :worker_died
+
+    #  receives a list of options that are used to determine the input files and output and input folders
+    #
+    # @param  [Hash]  options the options that can determine the input and output files and folders
+    # @option options [String] :input_dir The input directory
+    # @option options [String]:output_dir The output directory
+    # @option options [String] :taxonomy_file_name The taxonomy file name
+    # @option options [String] :destinations_file_name The destinations file name
+    #
+    # @see #work
+    #
+    # @return [void]
+    #
+    # @api public
+    def initialize(options = {})
+      Celluloid.boot
+      @options = options.is_a?(Hash) ? options.symbolize_keys : {}
+      @worker_supervisor = Celluloid::SupervisionGroup.run!
+      @workers = @worker_supervisor.pool(TaxGenerator::FileCreator, as: :workers, size: 50)
+      Actor.current.link @workers
+      @jobs = {}
+      @job_to_worker = {}
+      @worker_to_job = {}
+    end
+
+    #  returns the input folder from the options list
+    # otherwise the default path
+    #
+    # @return [String]
+    #
+    # @api public
+    def input_folder
+      @options.fetch(:input_dir, "#{root}/data/input")
+    end
+
+    #  returns the taxonomy filename from the option list
+    # otherwise the default filename
+    #
+    # @return [String]
+    #
+    # @api public
+    def taxonomy_file_name
+      @options.fetch(:taxonomy_filename, 'taxonomy.xml')
+    end
+
+    #  returns the destinations filename from the option list
+    # otherwise the default filename
+    #
+    # @return [String]
+    #
+    # @api public
+    def destinations_file_name
+      @options.fetch(:destinations_filename, 'destinations.xml')
+    end
+
+    #  returns the output folder path from the option list
+    # otherwise the default path
+    #
+    # @return [String]
+    #
+    # @api public
+    def output_folder
+      @options.fetch(:output_dir, "#{root}/data/output")
+    end
+
+    #  returns the full path to the taxonomy file
+    #
+    # @return [String]
+    #
+    # @api public
+    def taxonomy_file_path
+      File.join(input_folder, taxonomy_file_name)
+    end
+
+    #  returns the full path to the destinations file
+    #
+    # @return [String]
+    #
+    # @api public
+    def destinations_file_path
+      File.join(input_folder, destinations_file_name)
+    end
+
+    #  returns the full path to the static folder
+    #
+    # @return [String]
+    #
+    # @api public
+    def static_output_dir
+      File.join(output_folder, 'static')
+    end
+
+    # cleans the output folder and re-creates it and the static folder
+    #
+    # @return [void]
+    #
+    # @api public
+    def prepare_output_dirs
+      FileUtils.rm_rf Dir["#{output_folder}/**/*"]
+      create_directories(output_folder, static_output_dir)
+      FileUtils.cp_r(Dir["#{File.join(root, 'templates', 'static')}/*"], static_output_dir)
+    end
+
+    #  checks if all workers finished and returns true or false
+    #
+    # @return [Boolean]
+    #
+    # @api public
+    def all_workers_finished?
+      @jobs.all? { |_job_id, job| job['status'] == 'finished' }
+    end
+
+    #  registers all the jobs so that the managers can have access to them at any time
+    #
+    # @param  [Array] jobs the jobs that will be registered
+    #
+    # @return [void]
+    #
+    # @api public
+    def register_jobs(*jobs)
+      jobs.pmap do |job|
+        job = job.stringify_keys
+        @jobs[job['atlas_id']] = job
+      end
+    end
+
+    #  registers all the jobs, and then delegates them to workers
+    # @see #register_jobs
+    # @see TaxGenerator::FileCreator#work
+    #
+    # @param  [Array] jobs the jobs that will be delegated to the workers
+    #
+    # @return [void]
+    #
+    # @api public
+    def delegate_job(*jobs)
+      # jobs need to be added into the manager before starting task to avoid adding new key while iterating
+      register_jobs(*jobs)
+      current_actor = Actor.current
+      @jobs.pmap do |_job_id, job|
+        @workers.async.work(job, current_actor) if @workers.alive?
+      end
+    end
+
+    #  parses the destinations xml document, gets each destination and adds a new job for that
+    # destination in the job list and then returns it
+    # @see #nokogiri_xml
+    #
+    # @return [Array<Hash>]
+    #
+    # @api public
+    def fetch_file_jobs
+      jobs = [{ atlas_id: 0, taxonomy: @taxonomy, destination: nil, output_folder: output_folder }]
+      nokogiri_xml(destinations_file_path).xpath('//destination').pmap do |destination|
+        atlas_id = destination.attributes['atlas_id']
+        jobs << { atlas_id: atlas_id.value, taxonomy: @taxonomy, destination: destination, output_folder: output_folder }
+      end
+      jobs
+    end
+
+    #  fetches the jobs for file generation, then delegates the jobs to workers and waits untill workers finish
+    # @see #fetch_file_jobs
+    # @see #delegate_job
+    # @see #wait_jobs_termination
+    #
+    # @return [void]
+    #
+    # @api public
+    def generate_files
+      @condition = Celluloid::Condition.new
+      jobs = fetch_file_jobs
+      delegate_job(*jobs)
+      wait_jobs_termination
+    end
+
+    #  retrieves the information about the node from the tree and generates for each destination a new File
+    # @see #create_file
+    #
+    # @param  [TaxGenerator::TaxonomyTree]  taxonomy the taxonomy tree that will be used for fetching node information
+    #
+    # @return [void]
+    #
+    # @api public
+    def wait_jobs_termination
+      result = @condition.wait
+      return unless result.present?
+      terminate
+    end
+
+    #  registers the worker so that the current actor has access to it at any given time and starts the worker
+    # @see TaxGenerator::FileCreator#start_work
+    #
+    # @param  [Hash]  job the job that the worker will work
+    # @param  [TaxGenerator::FileCreator]  worker the worker that will create the file
+    #
+    # @return [void]
+    #
+    # @api public
+    def register_worker_for_job(job, worker)
+      @job_to_worker[job['atlas_id']] = worker
+      @worker_to_job[worker.mailbox.address] = job
+      log_message("worker #{worker.job_id} registed into manager")
+      Actor.current.link worker
+      worker.async.start_work
+    end
+
+    # generates the taxonomy tree , prints it and generates the files
+    # @see TaxGenerator::TaxonomyTree#new
+    # @see Tree::TreeNode#print_tree
+    # @see #generate_files
+    #
+    # @return [void]
+    #
+    # @api public
+    def work
+      prepare_output_dirs
+      if File.directory?(input_folder) && File.file?(taxonomy_file_path) && File.file?(destinations_file_path)
+        @taxonomy = TaxGenerator::TaxonomyTree.new(taxonomy_file_path)
+        @taxonomy.print_tree
+        generate_files
+      else
+        log_message('Please provide valid options', log_method: 'fatal')
+      end
+    end
+
+    # logs the message about working being dead if a worker crashes
+    # @param  [TaxGenerator::FileCreator]  worker the worker that died
+    # @param  [String]  reason the reason for which the worker died
+    #
+    # @return [void]
+    #
+    # @api public
+    def worker_died(worker, reason)
+      mailbox_address = worker.mailbox.address
+      job = @worker_to_job.delete(mailbox_address)
+      return if reason.blank? || job.blank?
+      log_message("worker job #{job['atlas_id']} with mailbox #{mailbox_address.inspect} died  for reason:  #{log_error(reason)}", log_method: 'fatal')
+    end
+  end
+end

data/lib/tax_generator/classes/taxonomy_tree.rb

@@ -0,0 +1,97 @@
+require_relative '../helpers/application_helper'
+module TaxGenerator
+  # class used to create the Taxonomy Tree
+  #
+  # @!attribute root_node
+  #   @return [Tree::TreeNode] the root node of the tree
+  #
+  # @!attribute document
+  #   @return [Nokogiri::XML] the xml document used to build the tree
+  class TaxonomyTree
+    include TaxGenerator::ApplicationHelper
+    attr_reader :root_node, :document
+
+    #  receives a file path that will be parsed and used to build the tree
+    # @see Tree::TreeNode#new
+    # @see #add_node
+    #
+    # @param  [String]  file_path the path to the xml file that will be parsed and used to build the tree
+    #
+    # @return [void]
+    #
+    # @api public
+    def initialize(file_path)
+      @document = nokogiri_xml(file_path)
+      taxonomy_root = @document.at_xpath('//taxonomy_name')
+      @root_node = Tree::TreeNode.new(taxonomy_root.content, nil)
+      @document.xpath('//node').pmap do |taxonomy_node|
+        add_node(taxonomy_node, @root_node)
+      end
+    end
+
+    #  gets the atlas_id from the nokogiri element and then searches first child whose name is 'node_name'
+    # and uses this to insert the node
+    # @see #insert_node
+    #
+    # @param  [Nokogiri::Element]  taxonomy_node the nokogiri element that wants to be added to the tree
+    # @param  [Tree::TreeNode]  node the parent node to which the element needs to be added
+    #
+    # @return [void]
+    #
+    # @api public
+    def add_taxonomy_node(taxonomy_node, node)
+      atlas_node_id = taxonomy_node.attributes['atlas_node_id']
+      node_name = taxonomy_node.children.find { |child| child.name == 'node_name' }
+      insert_node(atlas_node_id, node_name, node)
+    end
+
+    #  inserts a new node in the tree by checking first if atlas_id and node_name are present
+    # and then adds the node as child to the node passed as third argument
+    # @see Tree::TreeNode#new
+    #
+    # @param  [Nokogiri::Element]  atlas_node_id the element that holds the value of the atlas_id attribute
+    # @param  [Nokogiri::Element]  node_name the the element that holds the node name of the element
+    # @param  [Tree::TreeNode]  node the parent node to which the element needs to be added
+    #
+    # @return [void]
+    #
+    # @api public
+    def insert_node(atlas_node_id, node_name, node)
+      return if atlas_node_id.blank? || node_name.blank?
+      current_node = Tree::TreeNode.new(atlas_node_id.value, node_name.content)
+      node << current_node
+      current_node
+    end
+
+    #  checks to see if the nokogiri element has any childrens, if it has , will add it to the tree and iterates over the
+    # children and adds them as child to the newly added node
+    # @see #add_taxonomy_node
+    #
+    # @param  [Nokogiri::Element]  taxonomy_node the nokogiri element that wants to be added to the tree
+    # @param  [Tree::TreeNode]  node the parent node to which the element needs to be added
+    #
+    # @return [void]
+    #
+    # @api public
+    def add_node(taxonomy_node, node)
+      return unless taxonomy_node.children.any?
+      tax_node = add_taxonomy_node(taxonomy_node, node)
+      taxonomy_node.xpath('./node').pmap do |child_node|
+        add_taxonomy_node(child_node, tax_node) if tax_node.present?
+      end
+    end
+
+    #  receives a file path that will be parsed and used to build the tree
+    #
+    # @param  [String]  name the name of the method that is invoked against the tree
+    # @param  [Array]  args the arguments to the method
+    # @param  [Proc]  block the block that will be passed to the method
+    #
+    # @return [void]
+    #
+    # @api public
+    def method_missing(name, *args, &block)
+      @root_node.send name, *args, &block
+    end
+  end
+end

data/lib/tax_generator/cli.rb

@@ -0,0 +1,14 @@
+require_relative './all'
+module TaxGenerator
+  # this is the class that will be invoked from terminal , and willl use the invoke task as the primary function.
+  class CLI
+    # method used to start
+    #
+    # @return [void]
+    #
+    # @api public
+    def self.start
+      TaxGenerator::Application.new.run
+    end
+  end
+end

data/lib/tax_generator/helpers/application_helper.rb

@@ -0,0 +1,154 @@
+module TaxGenerator
+  # class that holds the helper methods used in the classes
+  module ApplicationHelper
+    delegate :app_logger,
+             to: :'TaxGenerator::Application'
+
+  module_function
+
+    # returns the text from a nokogiri element by rejecting blank elements
+    #
+    # @param  [Nokogiri::Element]  element the nokogiri element that will select only children with content and returns their text
+    #
+    # @return [String]
+    #
+    # @api public
+    def elements_with_content(element)
+      if element.present?
+        element.select { |elem| elem.content.present? }.join(&:text)
+      else
+        element
+      end
+    end
+
+    # returns the root path of the gem
+    #
+    # @return [void]
+    #
+    # @api public
+    def root
+      File.expand_path(File.dirname(File.dirname(File.dirname(__dir__))))
+    end
+
+    # returns a Nokogiri XML document from a file
+    #
+    # @param  [String]  file_path the path to the xml file that will be parsed
+    #
+    # @return [void]
+    #
+    # @api public
+    def nokogiri_xml(file_path)
+      Nokogiri::XML(File.open(file_path), nil, 'UTF-8')
+    end
+
+    # creates directories from a list of arguments
+    #
+    # @param  [Array]  args the arguments that will be used as directory names and will create them
+    #
+    # @return [void]
+    #
+    # @api public
+    def create_directories(*args)
+      args.pmap do |argument|
+        FileUtils.mkdir_p(argument) unless File.directory?(argument)
+      end
+    end
+
+    # sets the exception handler for celluloid actors
+    #
+    #
+    # @return [void]
+    #
+    # @api public
+    def set_celluloid_exception_handling
+      Celluloid.logger = app_logger
+      Celluloid.task_class = Celluloid::TaskThread
+      Celluloid.exception_handler do |ex|
+        unless ex.is_a?(Interrupt)
+          log_error(ex)
+        end
+      end
+    end
+
+    # Reads a file and interpretes it as ERB
+    #
+    # @param  [String]  file_path the file that will be read and interpreted as ERB
+    #
+    # @return [String]
+    #
+    # @api public
+    def erb_template(file_path)
+      template = ERB.new(File.read(file_path))
+      template.filename = file_path
+      template
+    end
+
+    # Displays a error with fatal log level
+    # @see #format_error
+    # @see #log_message
+    #
+    # @param  [Exception]  exception the exception that will be formatted and printed on screen
+    #
+    # @return [String]
+    #
+    # @api public
+    def log_error(exception)
+      message = format_error(exception)
+      log_message(message, log_method: 'fatal')
+    end
+
+    # formats a exception to be displayed on screen
+    #
+    # @param  [String]  message the message that will be printed to the log file
+    # @param  [Hash]  options the options used to determine how to log the message
+    # @option options [String] :log_method The log method , by default debug
+    #
+    # @return [String]
+    #
+    # @api public
+    def log_message(message, options = {})
+      app_logger.send(options.fetch(:log_method, 'debug'), message)
+    end
+
+    # formats a exception to be displayed on screen
+    #
+    # @param  [Exception]  exception the exception that will be formatted and printed on screen
+    #
+    # @return [String]
+    #
+    # @api public
+    def format_error(exception)
+      message = "#{exception.class} (#{exception.respond_to?(:message) ? exception.message : exception.inspect}):\n"
+      message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code)
+      message << '  ' << exception.backtrace.join("\n  ") if exception.respond_to?(:backtrace)
+      message
+    end
+
+    #  wrapper to execute a block and rescue from exception
+    # @see #set_celluloid_exception_handling
+    # @see #rescue_interrupt
+    # @see #log_error
+    #
+    # @return [void]
+    #
+    # @api public
+    def execute_with_rescue
+      set_celluloid_exception_handling
+      yield if block_given?
+    rescue Interrupt
+      rescue_interrupt
+    rescue => error
+      log_error(error)
+    end
+
+    #  rescues from a interrupt error and shows a message
+    #
+    # @return [void]
+    #
+    # @api public
+    def rescue_interrupt
+      `stty icanon echo`
+      puts "\n Command was cancelled due to an Interrupt error."
+    end
+  end
+end

data/lib/tax_generator/version.rb

@@ -0,0 +1,27 @@
+# Returns the version of the gem  as a <tt>Gem::Version</tt>
+module TaxGenerator
+  #  it prints the gem version as a string
+  #
+  # @return [String]
+  #
+  # @api public
+  def self.gem_version
+    Gem::Version.new VERSION::STRING
+  end
+
+  # module used to generate the version string
+  # provides a easy way of getting the major, minor and tiny
+  module VERSION
+    # major release version
+    MAJOR = 0
+    # minor release version
+    MINOR = 0
+    # tiny release version
+    TINY = 1
+    # prelease version ( set this only if it is a prelease)
+    PRE = nil
+
+    # generates the version string
+    STRING = [MAJOR, MINOR, TINY, PRE].compact.join('.')
+  end
+end

data/lib/tax_generator.rb

@@ -0,0 +1 @@
+require 'tax_generator/all'

data/spec/lib/tax_generator/classes/destination_spec.rb

@@ -0,0 +1,62 @@
+# encoding:utf-8
+require 'spec_helper'
+describe TaxGenerator::Destination do
+  let(:default_processor) {TaxGenerator::Processor.new}
+  let(:str) { default_processor.destinations_file_path }
+
+  let(:destination_xml) { @xml ||= Nokogiri::XML(File.open(str), nil, 'UTF-8') }
+  let(:info_base) { './/practical_information/health_and_safety' }
+  before(:each) do
+    @destination = TaxGenerator::Destination.new(destination_xml)
+  end
+
+  context 'xpaths' do
+    it 'fetches the introduction' do
+      destination_xml.expects(:xpath).with('.//introductory/introduction/overview').returns(true)
+      @destination.introduction
+    end
+
+    it 'fetches the history' do
+      destination_xml.expects(:xpath).with('./history/history/history').returns(true)
+      @destination.history
+    end
+
+    ['dangers_and_annoyances', 'while_youre_there', 'before_you_go', 'money_and_costs/money'].each do |name|
+      it "fetches the practical_information with #{name}" do
+        destination_xml.stubs(:xpath).returns([])
+        destination_xml.expects(:xpath).with(".//practical_information/health_and_safety/#{name}").returns([])
+        @destination.practical_information
+      end
+    end
+
+    it 'fetches the transport' do
+      destination_xml.expects(:xpath).with('.//transport/getting_around').returns(true)
+      @destination.transport
+    end
+
+    it 'fetches the weather' do
+      destination_xml.expects(:xpath).with('.//weather').returns(true)
+      @destination.weather
+    end
+
+    it 'fetches the work_live_study' do
+      destination_xml.expects(:xpath).with('.//work_live_study').returns(true)
+      @destination.work_live_study
+    end
+  end
+
+  it 'returns the hash' do
+    expect(@destination.to_hash).to eq(
+      {
+        introduction: @destination.introduction,
+        history: @destination.history,
+        practical_information: @destination.practical_information,
+        transport:  @destination.transport,
+        weather: @destination.weather,
+        work_live_study: @destination.work_live_study
+      }.each_with_object({}) do |(key, value), hsh|
+        hsh[key] = elements_with_content(value)
+        hsh
+      end)
+  end
+end