readme-score 0.0.2

data/lib/readme-score/document.rb

@@ -0,0 +1,37 @@
+require 'readme-score/document/filter'
+require 'readme-score/document/metrics'
+require 'readme-score/document/loader'
+require 'readme-score/document/parser'
+require 'readme-score/document/score'
+
+module ReadmeScore
+  class Document
+    attr_accessor :html, :filter, :metrics
+
+    def self.load(url)
+      loader = Loader.new(url)
+      loader.load!
+      new(loader.html)
+    end
+
+    def initialize(html)
+      @html = html
+      @noko = Nokogiri::HTML.fragment(@html)
+      @metrics = Document::Metrics.new(html_for_analysis)
+    end
+
+    # @return [String] HTML string ready for analysis
+    def html_for_analysis
+      @filter ||= Document::Filter.new(@noko)
+      @filter.filtered_html!
+    end
+
+    def score
+      @score ||= Score.new(metrics)
+    end
+
+    def inspect
+      "#<#{self.class}>"
+    end
+  end
+end

data/lib/readme-score/document/filter.rb

@@ -0,0 +1,70 @@
+module ReadmeScore
+  class Document
+    class Filter
+      SERVICES = ["travis-ci.org", "codeclimate.com", "gemnasium.com", "cocoadocs.org", "readme-score-api.herokuapp.com"]
+
+      def initialize(noko_or_html)
+        @noko = Util.to_noko(noko_or_html, true)
+      end
+
+      def filtered_html!
+        remove_license!
+        remove_contact!
+        remove_service_images!
+
+        @noko.to_s
+      end
+
+      def remove_license!
+        remove_heading_sections_named("license")
+        remove_heading_sections_named("licensing")
+        remove_heading_sections_named("copyright")
+      end
+
+      def remove_contact!
+        remove_heading_sections_named("contact")
+        remove_heading_sections_named("author")
+        remove_heading_sections_named("credits")
+      end
+
+      def remove_service_images!
+        SERVICES.each {|service|
+          remove_anchor_images_containing_url(service)
+        }
+      end
+
+      private
+        def remove_heading_sections_named(prefix)
+          any_hit = false
+          selectors = (1..5).map {|i| "h#{i}"}
+          selectors.each { |h|
+            @noko.search(h).each { |heading|
+              if heading.content.downcase == prefix
+                # hit - remove everything until the next heading
+                while sibling = heading.next_sibling
+                  if sibling.name.downcase.start_with?(heading.name)
+                    break
+                  else
+                    sibling.remove
+                  end
+                end
+                heading.remove
+                any_hit = true
+                break
+              end
+            }
+          }
+          any_hit
+        end
+
+        def remove_anchor_images_containing_url(url_fragment)
+          @noko.search('a').each {|a|
+            href = a['href']
+            if href && href.downcase.include?(url_fragment.downcase)
+              a.remove unless a.search('img').empty?
+            end
+          }
+        end
+    end
+  end
+end

data/lib/readme-score/document/loader.rb

@@ -0,0 +1,88 @@
+require 'uri/http'
+
+require 'readme-score/document/loader/github_readme_finder'
+
+module ReadmeScore
+  class Document
+
+    class Loader
+      MARKDOWN_EXTENSIONS = %w{md mdown markdown}
+
+      attr_accessor :request, :markdown
+
+      def self.github_repo_name(url)
+        uri = URI.parse(url)
+        return nil unless ["github.com", "www.github.com"].include?(uri.host)
+        path_components = uri.path.split("/")
+        return nil if path_components.reject(&:empty?).count != 2
+        path_components[-2..-1].join("/")
+      end
+
+      def self.is_github_repo_slug?(possible_repo)
+        !!(/^(\w|-)+\/(\w|-|\.)+$/.match(possible_repo))
+      end
+
+      def self.is_url?(possible_url)
+        !!(/https?:\/\//.match(possible_url))
+      end
+
+      def self.markdown_url?(url)
+        MARKDOWN_EXTENSIONS.select {|ext| url.downcase.end_with?(".#{ext}")}.any?
+      end
+
+      attr_accessor :response
+
+      def initialize(url)
+        @url = url
+      end
+
+      def github_repo_name
+        Loader.github_repo_name(@url)
+      end
+
+      def load!
+        if github_repo_name
+          if ReadmeScore.use_github_api?
+            load_via_github_api!
+          else
+            # take a guess at the raw file name
+            load_via_github_approximation!
+          end
+        else
+          @markdown = Loader.markdown_url?(@url)
+          @response ||= Unirest.get @url
+        end
+      end
+
+      def load_via_github_api!
+        @markdown = false
+        @response ||= OpenStruct.new.tap {|o|
+          @@client ||= Octokit::Client.new(access_token: ReadmeScore.github_api_token)
+          o.body = @@client.readme(github_repo_name, :accept => 'application/vnd.github.html').force_encoding("UTF-8")
+        }
+      end
+
+      def load_via_github_approximation!
+        @github_approximation_url ||= GithubReadmeFinder.new(url).find_url
+        @markdown = Loader.markdown_url?(@github_approximation_url)
+        @response ||= Unirest.get @github_approximation_url
+      end
+
+      def html
+        if markdown?
+          parse_markdown(@response.body)
+        else
+          @response.body
+        end
+      end
+
+      def parse_markdown(markdown)
+        Parser.new(@response.body).to_html
+      end
+
+      def markdown?
+        @markdown == true
+      end
+    end
+  end
+end

data/lib/readme-score/document/loader/github_readme_finder.rb

@@ -0,0 +1,39 @@
+require 'net/http'
+
+module ReadmeScore
+  class Document
+    class Loader
+      class GithubReadmeFinder
+        POSSIBLE_README_FILES = %w{README.md readme.md README readme ReadMe ReadMe.md}
+
+        def initialize(github_repo_url)
+          @repo_url = github_repo_url
+        end
+
+        def find_url
+          uri = URI.parse(@repo_url)
+          uri.scheme = "https"
+          uri.host = "raw.githubusercontent.com"
+          original_path = uri.path
+          readme_url = nil
+          POSSIBLE_README_FILES.each {|f|
+            uri.path = File.join(original_path, "master/#{f}")
+            readme_url = uri.to_s if reachable?(uri.to_s)
+            break if readme_url
+          }
+          readme_url
+        end
+
+        private
+          def reachable?(url)
+            begin
+              RestClient::Request.execute(method: :head, url: url)
+              true
+            rescue RestClient::Exception
+              false
+            end
+          end
+      end
+    end
+  end
+end

data/lib/readme-score/document/metrics.rb

@@ -0,0 +1,125 @@
+module ReadmeScore
+  class Document
+    class Metrics
+      EQUATION_METRICS = [
+        :cumulative_code_block_length
+      ]
+      def initialize(noko_or_html)
+        @noko = Util.to_noko(noko_or_html)
+      end
+
+      def cumulative_code_block_length
+        all_code_blocks.inner_html.length
+      end
+
+      def number_of_links
+        all_links.length
+      end
+
+      def number_of_code_blocks
+        all_code_blocks.length
+      end
+
+      def number_of_paragraphs
+        all_paragraphs.length
+      end
+
+      def number_of_non_code_sections
+        (all_paragraphs + all_lists).length
+      end
+
+      def code_block_to_paragraph_ratio
+        if number_of_paragraphs.to_f == 0.0
+          return 0
+        end
+        number_of_code_blocks.to_f / number_of_paragraphs.to_f
+      end
+
+      def number_of_internal_links
+        all_links.select {|a|
+          internal_link?(a)
+        }.count
+      end
+
+      def number_of_external_links
+        all_links.reject {|a|
+          internal_link?(a)
+        }.count
+      end
+
+      def has_lists?
+        all_lists.length > 0
+      end
+
+      def has_images?
+        number_of_images > 0
+      end
+
+      def number_of_images
+        all_images.count
+      end
+
+      def has_gifs?
+        number_of_gifs > 0
+      end
+
+      def number_of_gifs
+        all_gifs.length
+      end
+
+      def has_tables?
+        !all_tables.empty?
+      end
+
+      def inspect
+        "#<#{self.class}>"
+      end
+
+      private
+        def all_links
+          @noko.search('a')
+        end
+
+        def all_code_blocks
+          @noko.search('pre')
+        end
+
+        def all_paragraphs
+          @noko.search('p')
+        end
+
+        def all_lists
+          @noko.search('ol') + @noko.search('ul')
+        end
+
+        def all_images
+          @noko.search('img')
+        end
+
+        def all_gifs
+          all_images.select {|a|
+            source_attributes = ['src', 'data-canonical-src']
+            source_attributes.map {|_attr|
+              a[_attr] && a[_attr].downcase.include?(".gif")
+            }.any?
+          }
+        end
+
+        def all_tables
+          @noko.search('table')
+        end
+
+        def internal_link?(a)
+          external_prefixes = %w{http}
+          href = a['href'].downcase
+
+          return true if href.include?("://github") || href.include?("github.io")
+
+          external_prefixes.select {|prefix|
+            href.start_with?(prefix)
+          }.any?
+        end
+
+    end
+  end
+end

data/lib/readme-score/document/parser.rb

@@ -0,0 +1,24 @@
+module ReadmeScore
+  class Document
+    class Parser
+      def initialize(markdown)
+        @markdown = markdown
+      end
+
+      def to_html
+        parser.render(@markdown)
+      end
+
+      def parser
+        @@parser ||= Redcarpet::Markdown.new(
+          Redcarpet::Render::HTML,
+          no_intra_emphasis: true,
+          autolink: true,
+          fenced_code_blocks: true,
+          tables: true,
+          strikethrough: true
+        )
+      end
+    end
+  end
+end

data/lib/readme-score/document/score.rb

@@ -0,0 +1,104 @@
+module ReadmeScore
+  class Document
+    class Score
+
+      SCORE_METRICS = [
+        {
+          metric: :number_of_code_blocks,
+          description: "Number of code blocks",
+          value_per: 5,
+          max: 40
+        },
+        {
+          metric: :number_of_non_code_sections,
+          description: "Number of non-code sections",
+          value_per: 5,
+          max: 30
+        },
+        {
+          metric: :has_lists?,
+          description: "Has any lists?",
+          value: 10
+        },
+        {
+          metric: :number_of_images,
+          description: "Number of images",
+          value_per: 5,
+          max: 15
+        },
+        {
+          metric: :number_of_gifs,
+          description: "Number of GIFs",
+          value_per: 5,
+          max: 15
+        },
+        {
+          metric: :cumulative_code_block_length,
+          description: "Amount of code",
+          value_per: 0.0009475244447271192,
+          max: 10
+        },
+        {
+          metric_name: :low_code_block_penalty,
+          description: "Penalty for lack of code blocks",
+          metric: :number_of_code_blocks,
+          if_less_than: 3,
+          value: -10
+        }
+      ]
+
+      attr_accessor :metrics
+
+      def initialize(metrics)
+        @metrics = metrics
+      end
+
+      def score_breakdown(as_description = false)
+        breakdown = {}
+        SCORE_METRICS.each { |h|
+          metric_option = OpenStruct.new(h)
+          metric_name = metric_option.metric_name || metric_option.metric
+          metric_score_value = 0
+          # points for each occurance
+          if metric_option.value_per
+            metric_score_value = [metrics.send(metric_option.metric) * metric_option.value_per, metric_option.max].min
+          elsif metric_option.if_less_than
+            if metrics.send(metric_option.metric) < metric_option.if_less_than
+              metric_score_value = metric_option.value
+            end
+          else
+            metric_score_value = metrics.send(metric_option.metric) ? metric_option.value : 0
+          end
+          if as_description
+            breakdown[metric_option.description] = [metric_score_value, metric_option.max || metric_option.value]
+          else
+            breakdown[metric_name] = metric_score_value
+          end
+        }
+        breakdown
+      end
+      alias_method :breakdown, :score_breakdown
+
+      def human_breakdown
+        score_breakdown(true)
+      end
+
+      def total_score
+        score = 0
+        score_breakdown.each {|metric, points|
+          score += points.to_i
+        }
+        [[score, 100].min, 0].max
+      end
+      alias_method :to_i, :total_score
+
+      def to_f
+        to_i.to_f
+      end
+
+      def inspect
+        "#<#{self.class} - #{total_score}>"
+      end
+    end
+  end
+end