abrupt 0.0.1

data/lib/abrupt.rb

@@ -0,0 +1,90 @@
+# @author Manuel Dudda
+Dir[File.dirname(__FILE__) + '/abrupt/*.rb'].each do |file|
+  require file
+end
+require 'pp'
+
+# Extension for String class
+class String
+  def remove_last_slashes
+    gsub(/([\/]*)$/, '')
+  end
+
+  def append_last_slash
+    gsub(/([^\/])$/, '\1/')
+  end
+end
+
+# Extension for all objects
+class Object
+  def ensure_to_a
+    [self].flatten.compact
+  end
+end
+
+# This module is cool
+# @abstract
+module Abrupt
+  VOC = RDF::Vocabulary.new('http://wba.cs.hs-rm.de/AbRUPt/')
+  VOC_FILE = File.join File.dirname(__dir__), 'assets', 'voc', 'tbox.ttl'
+  RULES_DIR = File.join File.dirname(__dir__), 'assets', 'rules', '*'
+  DELIMITER = '/'
+  PREFIXES = {
+      abrupt: VOC.to_s,
+      rdf: RDF.to_s,
+      rdfs: RDF::RDFS.to_s,
+      xsd: RDF::XSD.to_s,
+      owl: RDF::OWL.to_s
+  }
+
+  TIME_INPUT_FORMAT = '%d/%b/%Y:%H:%M:%S'
+  TIME_OUTPUT_FORMAT = '%Y-%m-%d_%H%M%S'
+
+  def self.parse_time(time)
+    DateTime.strptime(time, TIME_INPUT_FORMAT)
+  end
+
+  def self.format_time(time)
+    parse_time(time).strftime(TIME_OUTPUT_FORMAT)
+  end
+
+  def self.root
+    File.dirname __dir__
+  end
+
+  def self.log(msg)
+    print msg
+  end
+
+  def self.crawl(uri, *args)
+    opts = args.first
+    crawler = Abrupt::Crawler.new uri, opts
+    start_time = Time.now
+    log "begin: #{start_time}\n"
+    result = crawler.crawl
+    end_time = Time.now
+    log "\nfinished in #{(end_time - start_time).round} sec.\n\n"
+    case opts[:format]
+    when 'xml'
+      puts Converter.xml(result)
+    else # owl as default
+      puts Converter.owl(result)
+    end
+  end
+
+  def self.convert(file, *args)
+    converter = Converter.instance
+    assertions = args.last[:assertions].split ','
+    converter.init(args[1]) # options
+    append file, args.first, assertions
+    converter.result
+  end
+
+  def self.append(file, user_file, assertions)
+    converter = Converter.instance
+    converter.append_tbox if assertions.include?('tbox')
+    converter.append_website_data(file) if assertions.include?('website')
+    converter.append_user_data(user_file) if assertions.include?('user')
+    converter.append_rules if assertions.include?('rules')
+  end
+end

data/lib/abrupt/converter.rb

@@ -0,0 +1,130 @@
+# @author Manuel Dudda
+require 'singleton'
+require 'rest_client'
+require 'gyoku'
+require 'rdf'
+require 'linkeddata'
+require 'active_support'
+require 'active_support/core_ext'
+Dir[File.dirname(__FILE__) + '/transformation/*.rb',
+    File.dirname(__FILE__) + '/transformation/website/*.rb',
+    File.dirname(__FILE__) + '/transformation/client/*.rb'].each do |file|
+  require file
+end
+
+# Abrupt Converter
+module Abrupt
+  # Converter
+  class Converter
+    include Singleton
+    include RDF
+    attr_accessor :hsh, :values, :result, :format, :uri
+
+    def init(options = {})
+      @format = options[:format].try(:to_sym) || :turtle
+      @result = Repository.new
+    end
+
+    def append_tbox
+      @result << Repository.load(VOC_FILE)
+    end
+
+    def append_website_data(hsh)
+      init_hsh(hsh)
+      @uri = URI(@hsh[:website][:domain])
+      init_website
+      perform
+    end
+
+    def init_website
+      domain = RDF::URI("#{VOC}Website/#{@uri}")
+      @result << Statement.new(domain, RDF.type, VOC.Website)
+      @result << Statement.new(domain, VOC.hostName, @uri.host)
+    end
+
+    def init_hsh(hsh)
+      hsh = Hash.from_xml(File.read(hsh)) unless hsh.is_a?(Hash)
+      @hsh = hsh.deep_symbolize_keys
+      return unless @hsh[:website]
+      @hsh[:website][:url].each_with_index do |value, i|
+        Transformation::Website::Base.subclasses.each do |transformation_class|
+          @hsh[:website][:url][i] =
+              transformation_class.customize_to_schema(value)
+        end
+      end
+    end
+
+    def self.xml(hsh)
+      Gyoku.xml hsh
+    end
+
+    def self.json(hsh)
+      hsh.to_json
+    end
+
+    def add_to_result(statements)
+      statements.each { |stmt| @result << stmt }
+    end
+
+    def perform
+      website = ['Website', @uri.to_s]
+      @hsh[:website][:url].each do |url|
+        page = ['Page', url[:name].append_last_slash]
+        page_transformator = Transformation::Base.new(website, page)
+        add_to_result page_transformator.add_individuals # add Page
+        next unless url[:state]
+        perform_states url[:state], website + page
+      end
+    end
+
+    def perform_states(states, parent_uri)
+      states = states.is_a?(Array) ? states : [states]
+      states.each do |value|
+        state = ['State', value[:name]]
+        # MAYBE empty?
+        add_to_result Transformation::Base.new(parent_uri, state).result
+        Transformation::Website::Base.subclasses.each do |transformation_class|
+          t = transformation_class.new(parent_uri + state, nil, value)
+          add_to_result t.add_individuals
+        end
+      end
+    end
+
+    def append_user_data(file)
+      return unless file.is_a?(String) && File.exist?(file)
+      xml = Hash.from_xml(File.read(file)).deep_symbolize_keys
+      xml[:database][:visitor].ensure_to_a.each do |values|
+        ip = values[:ip]
+        next unless ip
+        visitor = Transformation::Client::Visitor.new(
+            ['Website', @uri.to_s], ['Visitor', ip], values)
+        add_to_result visitor.add_individuals
+        append_pages_for_visitor(visitor)
+      end
+      @result
+    end
+
+    def append_pages_for_visitor(visitor)
+      pages = visitor.values[:pages][:page].ensure_to_a
+      pages.each do |page|
+        time = ::Abrupt.format_time(page[:entertime])
+        Transformation::Client::Base.subclasses.each do |transformation_class|
+          transformator = transformation_class.new(
+              visitor.parent_uri + visitor.uri,
+              ['Visit', time], page
+          )
+          add_to_result transformator.add_individuals
+        end
+      end
+    end
+
+    def append_rules
+      Dir.glob(RULES_DIR).each do |rule_directory|
+        Dir.glob(File.join(rule_directory, '*')).each do |rule_file|
+          rule = Repository.load(rule_file)
+          add_to_result(rule.statements)
+        end
+      end
+    end
+  end
+end

data/lib/abrupt/crawler.rb

@@ -0,0 +1,125 @@
+# @author Manuel Dudda
+require 'rest_client'
+require 'addressable/uri'
+%w( base
+    readability
+    subject
+    input
+    complexity
+    picture
+    link
+    absolute_url).each do |f|
+  require_relative "service/#{f}"
+end
+module Abrupt
+  # Crawler for a website including all followed urls
+  # with performing abrupt services
+  # BETA!!!
+  class Crawler
+    SERVICE_MAPPING = {
+        r: Service::Readability,
+        i: Service::Input,
+        s: Service::Subject,
+        c: Service::Complexity,
+        l: Service::Link,
+        p: Service::Picture
+    }
+
+    def initialize(uri, *args)
+      @uri = Addressable::URI.parse(uri).normalize
+      opts = args.first
+      @options = {
+          lang: 'en',
+          services: %w(r i s c l p),
+          depth: '3',
+          word_limit: 20
+      }
+      @options[:services] = opts[:services] if opts[:services]
+      @options[:lang] = opts[:lang] if opts[:lang]
+      @follow_links = !opts[:nofollow]
+      @result = {}
+    end
+
+    # Crawls a page, saves the service results in result hash
+    # and returns an array with the existing uris of this page.
+    #
+    # @param uri [String] the uri to crawl
+    # @return [JSON] result
+    def crawl(uri = nil)
+      Abrupt.log '.'
+      uri ||= @uri.to_str.append_last_slash
+      unless @result[uri]
+        html = fetch_html(uri)
+        @result[uri] ||= {}
+        @result[uri] = perform_services(html) if html
+        # new_uris.select! { |url| same_host?(url) } # filter
+        uris_with_same_host(uri).uniq.each { |url| crawl(url) } if @follow_links
+      end
+      Service::Base.transform_hash(@result)
+    end
+
+    # TODO: maybe as class method
+    def uris_with_same_host(uri)
+      if @result[uri][:link] && @result[uri][:link]['a']
+        @result[uri][:link]['a'].to_a.map do |link|
+          link['href'] if same_host?(link['href'])
+        end.compact
+      else
+        []
+      end
+    end
+
+    def fetch_html(uri)
+      uri = Addressable::URI.parse(uri.strip).normalize.to_str
+      begin
+        response = ::RestClient.get uri, accept: :html
+        content_type = response.headers[:content_type].to_s
+        case response.code
+        when 200...400
+          response.to_str if html?(content_type)
+        else
+          false
+        end
+      rescue => e
+        puts "error fetching html on #{uri}"
+        puts e
+        nil
+      end
+    end
+
+    def html?(content_type)
+      content_type.start_with?('text/html')
+    end
+
+    def same_host?(uri)
+      !uri.to_s.empty? && Addressable::URI.parse(uri).host.eql?(@uri.host)
+    end
+
+    def init_services_hash(html)
+      @options[:services].map do |s|
+        s = s.to_sym
+        service_class = SERVICE_MAPPING[s]
+        available_options = service_class.available_options
+        opts = available_options.map { |o| [o, @options[o.to_sym]] }.to_h
+        service = service_class.new(html, opts)
+        [service_class.keyname, service]
+      end.to_h
+    end
+
+    def canonize_html(html)
+      baseurl = "#{@uri.scheme}://#{@uri.host}"
+      converter = Service::AbsoluteUrl.new(html, baseurl: baseurl)
+      converter.execute
+    end
+
+    def perform_services(html)
+      result = {}
+      html = canonize_html(html)
+      services_hash = init_services_hash(html)
+      services_hash.each do |json_field, service_class|
+        result[json_field.to_sym] = service_class.execute
+      end
+      result
+    end
+  end
+end

data/lib/abrupt/service/absolute_url.rb

@@ -0,0 +1,32 @@
+# @author Manuel Dudda
+module Abrupt
+  module Service
+    # Complexity service
+    # documentation see 'http://wba.cs.hs-rm.de/AbRUPt/service/absoluteurl/'
+    class AbsoluteUrl < Base
+      # TODO: outsource service uri to module Service
+      SERVICE_URI = 'http://wba.cs.hs-rm.de/AbRUPt/service/absoluteurl/'
+
+      def service_uri
+        SERVICE_URI
+      end
+
+      def execute
+        options = {
+            method: :post,
+            timeout: 6000,
+            open_timeout: 6000,
+            accept: :html
+        }
+        options.merge!(url: @url, payload: @html)
+        begin
+          RestClient::Request.execute(options).to_str
+        rescue => e
+          puts "some problems with #{@url}"
+          puts e
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/abrupt/service/base.rb

@@ -0,0 +1,75 @@
+# @author Manuel Dudda
+require 'rest_client'
+module Abrupt
+  module Service
+    # base class
+    class Base
+      attr_accessor :url, :abbr, :options, :response
+      # TODO: outsource service uri to module Service
+      SERVICE_URI = 'http://wba.cs.hs-rm.de/AbRUPt/service/complexity/public/index.php/api/v1/complexity'
+
+      def service_uri
+        SERVICE_URI
+      end
+
+      def initialize(html, options = {})
+        @html = html
+        @options = options
+        query_params = if @options.count > 0
+                         options_arr = @options.map { |k, v| "#{k}=#{v}" }
+                         '?' + options_arr.reduce { |a, e| "#{a}&#{e}" }
+                       else
+                         ''
+                       end
+        @url = service_uri + query_params
+        @abbr = self.class.name[0].downcase
+        @options = []
+      end
+
+      def self.available_options
+        []
+      end
+
+      def self.keyname
+        name.split('::').last.downcase
+      end
+
+      # TODO: naming of interface execute
+      def execute
+        options = {
+            method: :post,
+            timeout: 6000,
+            open_timeout: 6000,
+            accept: :schema
+        }
+        options.merge!(url: @url, payload: @html)
+        begin
+          res = RestClient::Request.execute(options).to_str
+          @response = JSON.parse(res)
+        rescue => e
+          puts "some problems with #{@url}"
+          puts e
+          nil
+        end
+      end
+
+      def self.transform_hash(hsh)
+        uri = Addressable::URI.parse(hsh.keys.first).normalize
+        result = {
+            website: {
+                domain: "#{uri.scheme}://#{uri.host}",
+                url: []
+            }
+        }
+        hsh.each_with_index do |(key, value), _index|
+          page = {
+              name: key,
+              state: value
+          }
+          result[:website][:url] << page
+        end
+        result.deep_symbolize_keys
+      end
+    end
+  end
+end