sla 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5fdee1ea51728193de0bfeaa1b97b21c024f023f
+  data.tar.gz: 5cf1689f5408018346cd9cc3248e40cb7825ee5e
+SHA512:
+  metadata.gz: e812c02cd97d3b351f0f8ff988b9e6163d8d0c569ecd88048d5a36412bd8acc211bd9c24db3ad750b9ee345f3fe3d5fbbc1f859aab1858a45109941acea255cd
+  data.tar.gz: f37e16e97dd8997e1a51f460d85b8fcd3775902550156263d5955bf78956e6069548d7650cba2916d15791ae9c298b4a47e305a351229b3e9580d5d3d8d85846

data/README.md

@@ -0,0 +1,3 @@
+Site Link Analyzer
+==================================================
+

data/lib/sla/base.rb

@@ -0,0 +1,17 @@
+module SLA
+  class Base
+    attr_accessor :domain
+
+    def cache
+      @cache ||= Cache.instance.cache
+    end
+
+    def url_manager
+      @url_manager ||= UrlManager.instance
+    end
+
+    def base_url
+      url_manager.base_url.to_s
+    end
+  end
+end

data/lib/sla/cache.rb

@@ -0,0 +1,15 @@
+module SLA
+  class Cache
+    include Singleton
+
+    def cache
+      @cache ||= cache!
+    end
+    
+    def cache! 
+      result = WebCache.new
+      result.life = 60 * 60 * 24 # 24 hours
+      result
+    end
+  end
+end

data/lib/sla/checker.rb

@@ -0,0 +1,45 @@
+module SLA
+  class Checker < Base
+    include Colsole
+
+    attr_accessor :checked_links, :results, :max_depth, :next_check
+
+    def initialize
+      @max_depth = 10
+      @checked_links = []
+      @next_check ||= []
+    end
+
+    def count
+      checked_links.count
+    end
+
+    def on_check(urls=nil, depth=1, &block)
+      urls ||= [base_url]
+
+      self.next_check = []
+
+      urls.each do |url|
+        check_url url, depth, &block
+        if depth < max_depth
+          on_check next_check, depth+1, &block
+        end
+      end
+    end
+
+    def check_url(url, depth, &block)
+      page = Page.new url, depth: depth, base_url: base_url
+      page.validate
+
+      yield page
+      return if depth >= max_depth
+
+      page.links.each do |link|
+        next if checked_links.include? link.url
+        
+        checked_links.push link.url
+        next_check.push link.url
+      end
+    end
+  end
+end

data/lib/sla/command_line.rb

@@ -0,0 +1,68 @@
+module SLA
+  class CommandLine < Base
+    include Singleton
+    include Colsole
+
+    def execute(argv=[])
+      doc = File.read File.dirname(__FILE__) + '/docopt.txt'
+      begin
+        args = Docopt::docopt(doc, argv: argv, version: VERSION)
+        handle args
+      rescue Docopt::Exit => e
+        puts e.message
+      end
+    end
+
+    def handle(args)
+      if args['check']
+        @domain     = args['DOMAIN']
+        @max_depth  = args['--depth'].to_i
+        @cache_life = args['--cache'].to_i
+        check_domain
+      end
+    end
+
+    def check_domain
+      checker = Checker.new
+      checker.max_depth    = @max_depth
+      checker.cache.life   = @cache_life
+      url_manager.base_url = @domain
+
+      File.unlink 'log.log' if File.exist? 'log.log'
+
+      count = 1
+      failed = 0
+
+      open('log.log', 'a') do |f|
+        checker.on_check do |page|
+          indent = '-' * page.depth
+
+          status = page.status
+          colored_status = color_status status
+          failed +=1 if status != '200'
+
+          say    "#{count} #{colored_status} #{indent} #{page.name}"
+          f.puts "#{count} #{status} #{indent} #{page.name}"
+          count += 1
+        end
+
+        color = failed > 0 ? '!txtred!' : '!txtgrn!'
+        say "#{color}Done with #{failed} failures"
+        f.puts "Done with #{failed} failures"
+      end
+
+    end
+
+    def color_status(status)
+      case status
+      when '200'
+        '!txtgrn!200!txtrst!'
+      when '404'
+        '!txtred!404!txtrst!'
+      else
+        status
+      end
+    end
+
+  end
+end

data/lib/sla/docopt.txt

@@ -0,0 +1,20 @@
+SLA
+
+Usage:
+  sla check DOMAIN [options]
+  sla (-h|--help|--version)
+
+Commands:
+  check
+    Start checking for broken links on a given domain
+  
+Options:
+  --depth, -d DEPTH
+    Set crawling depth [default: 5]
+
+  --cache, -c LIFE
+    Set cache life in seconds [default: 86400]
+
+Examples:
+  sla check example.com
+  sla check example.com -c360 -d10

data/lib/sla/link.rb

@@ -0,0 +1,15 @@
+module SLA
+  class Link
+    attr_accessor :text, :url
+
+    def initialize(text, url)
+      @text = text
+      @url = url
+    end
+
+    def path
+      uri = URI.parse url
+      uri.request_uri
+    end
+  end
+end

data/lib/sla/page.rb

@@ -0,0 +1,75 @@
+module SLA
+  class Page < Base
+    attr_accessor :depth, :status, :base_uri
+    attr_reader :url
+
+    def initialize(url, opts={})
+      @url = url
+      @base_uri = url
+      @status = '000'
+      self.depth = opts[:depth] if opts[:depth]
+    end
+
+    def valid?
+      content
+      status == '200'
+    end
+
+    def validate
+      content
+    end
+
+    def name
+      @name ||= name!
+    end
+
+    def name!
+      uri = URI.parse url
+      if uri.request_uri.empty? || uri.request_uri == '/'
+        url 
+      else
+        uri.request_uri
+      end
+    end
+
+    def protocol
+      @protocol ||= protocol!
+    end
+
+    def protocol!
+      uri = URI.parse url
+      uri.scheme
+    end
+
+    def content
+      @content ||= content!
+    end
+
+    def content!
+      response = cache.get url
+      self.status = response.error ? '404' : '200'
+      self.base_uri = response.base_uri
+      response.content
+    end
+
+    def doc
+      @doc ||= Nokogiri::HTML content
+    end
+
+    def links
+      @links ||= links!
+    end
+
+    def links!
+      links = doc.css('a')
+      result = []
+      links.each do |link|
+        href = url_manager.absolute link['href'], base_uri
+        next unless href
+        result.push Link.new link.text, href
+      end
+      result
+    end
+  end
+
+end

data/lib/sla/url_manager.rb

@@ -0,0 +1,23 @@
+module SLA
+  class UrlManager < Base
+    include Singleton
+
+    attr_reader :uri, :base_url
+
+    def base_url=(url)
+      url = "http://#{url}" unless url[0..3] == 'http'
+      response = cache.get url
+      @base_url = response.base_uri
+    end
+
+    def absolute(relative, base=nil)
+      return false if relative =~ /^(tel|mailto|#)/
+      base ||= base_url
+      relative = URI.encode relative
+      result = URI.join(base, relative)
+
+      return false unless result.host == base_url.host
+      result.to_s
+    end
+  end
+end

data/lib/sla/version.rb

@@ -0,0 +1,3 @@
+module SLA
+  VERSION = "0.0.1"
+end

data/lib/sla.rb

@@ -0,0 +1,20 @@
+require 'webcache'
+require 'nokogiri'
+require 'colsole'
+require 'singleton'
+require 'docopt'
+require 'uri'
+
+require 'sla/version'
+require 'sla/base'
+require 'sla/page'
+require 'sla/link'
+require 'sla/checker'
+require 'sla/command_line'
+require 'sla/cache'
+require 'sla/url_manager'
+
+# Remove before deploy
+require 'byebug'
+require 'awesome_print'
+require 'pp'

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: sla
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Danny Ben Shitrit
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: runfile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: colsole
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: webcache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: runfile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: adsf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: runfile-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: Check for broken links on a website
+email: db@dannyben.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/sla.rb
+- lib/sla/base.rb
+- lib/sla/cache.rb
+- lib/sla/checker.rb
+- lib/sla/command_line.rb
+- lib/sla/docopt.txt
+- lib/sla/link.rb
+- lib/sla/page.rb
+- lib/sla/url_manager.rb
+- lib/sla/version.rb
+homepage: https://github.com/DannyBen/sla
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Site Link Analyzer
+test_files: []