pageinfo 0.1.0 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13de38291ef72f6b4e26b68d8dab4d883957ba02
-  data.tar.gz: 27c1569ddf15e528c4efe8428e14d66a78393bda
+  metadata.gz: 551cee8d620c763efb8351f36e05c467294b1fc2
+  data.tar.gz: d3101badc2494708e3867afd031a605f3ac63489
 SHA512:
-  metadata.gz: b5e192e0e2751f1d0a3d9d924b20a3fc06029e147a3624e555640013ee2fe95290a9c76e2a7bb5d4a1a0d01865f9d2cf5047ee330a0cb0c1f9fc028cdb8c58d0
-  data.tar.gz: 96eb29d6ff9bc10c339a23e93892edd331270d7572496ba2a0e4fb3050ccf7e656a219a0948a21b9b4cfcb76c21e0d1ec265695429f856f56e88d2edd931fd7d
+  metadata.gz: 2f53e5eb1f18d7098f56f0b282a03b3339da1c3a5d8ab2eec2f786c4c86ea6fc75e779178deebd14264b3073ea7f7376873e0f82b4e9464280545f819772a201
+  data.tar.gz: a8e641eadfc3174532177e5324e2127913d625129e2c679b9a685abf26ac23efdb90011a26e49d85ffd9f9d75c9d3d89ef40c52d5f57a930ade30c190f06051f

data/README.md

@@ -1,8 +1,6 @@
 # Pageinfo
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pageinfo`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+Pageinfo will browse through your site then list every page on your site and give detail meta info about the page.
 
 ## Installation
 
@@ -22,18 +20,20 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+Browse your site using:
+
+    $ pageinfo "http://www.yoursite.com"
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+<!-- After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). -->
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/pageinfo/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
+1. Fork it ( https://github.com/aditiamahdar/pageinfo/fork )
+2. Create your feature branch (`git checkout -b new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
+4. Push to the branch (`git push origin new-feature`)
 5. Create a new Pull Request

data/bin/pageinfo

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'pageinfo'
+Pageinfo.detect ARGV[0]

data/exe/pageinfo

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'pageinfo'
+Pageinfo.detect ARGV[0]

data/lib/pageinfo.rb

@@ -1,8 +1,132 @@
 require "pageinfo/version"
+require "uri"
+require "typhoeus"
+require "nokogiri"
 
 module Pageinfo
-  def self.process(str)
-    # TODO: process `str`
-    str
+  def self.hi
+    greeting = "Hello World!"
+    puts greeting
+    greeting
   end
+
+  def self.detect(url)
+    content = ["url", "status", "time", "title", "description", "keyword"].join(",")
+    content << new_line
+
+    @@no = 0
+    @@main_host = get_host(URI.parse(url))
+    scrapped_links, scrapped_urls = [url], [url]
+
+    conn = Typhoeus.get(url)
+    page = Nokogiri::HTML(conn.body)
+
+    content << get_info(conn, page)
+    content << new_line
+
+    @links = get_page_links(page)
+    puts "Homepage links: #{@links.count}"
+    while true do
+      if link = @links.shift
+        full_url = get_full_url(link)
+        unless full_url.nil?
+          if (scrapped_urls & [full_url, "#{full_url}/", "#{full_url}/#"]).empty?
+            # No duplicate link
+            conn = Typhoeus.get(full_url)
+            page = Nokogiri::HTML(conn.body)
+            content << get_info(conn, page)
+            content << new_line
+
+            scrapped_links << link
+            scrapped_urls << full_url
+
+            new_links = get_page_links(page)
+            new_links = new_links - @links
+            new_links = new_links - scrapped_links
+            @links = @links + new_links unless new_links.empty?
+            puts "Links: #{@links.count} left, #{new_links.count} new links"
+          end
+        end
+      else
+        break;
+      end
+    end
+
+    File.open("pageinfo.csv", "w") { |file| file.write content }
+  end
+
+  private
+    def self.new_line
+      "\n"
+    end
+
+    def self.get_host(uri)
+      (uri.port.eql?(443) ? "https://" : "http://") +
+      uri.host +
+      (uri.port.eql?(80) ? "" : ":#{uri.port}")
+    end
+
+    def self.get_info(conn, page)
+      @@no = @@no.next
+      puts "#{@@no}. #{conn.effective_url}"
+      [
+        "\"#{conn.effective_url}\"",
+        "\"#{conn.response_code}\"",
+        "\"#{conn.total_time}\"",
+        "\"#{get_head(page, "title")}\"",
+        "\"#{get_head(page, "description")}\"",
+        "\"#{get_head(page, "keywords")}\"",
+      ].join(",")
+    end
+
+    def self.get_head(page, type)
+      case type
+      when "title"
+        page.at("title").text.strip rescue ""
+      when "description"
+        page.at("meta[name=description]").attribute("content").value.strip rescue ""
+      when "keywords"
+        page.at("meta[name=keywords]").attribute("content").value.strip rescue ""
+      end
+    end
+
+    def self.get_full_url(link)
+      if bad_link?(link)
+        nil
+      elsif link.match(/^\//)
+        "#{@@main_host}#{valid_link(link)}"
+      elsif link.match(@@main_host)
+        valid_link(link)
+      else
+        "#{@@main_host}/#{link}"
+      end
+    end
+
+    def self.bad_link?(link)
+      [nil, "#"].include?(link) ||
+      link.match(/^javascript/) ||
+      (link.match(/^http/) && external_link?(link))
+    end
+
+    def self.external_link?(link)
+      uri = URI.parse(link) rescue nil
+      uri.nil? ? true : !@@main_host.eql?(get_host(uri))
+    end
+
+    def self.valid_link(link)
+      if link.match(/\/$/)
+        link[0..-2]
+      elsif link.match(/\/\#$/)
+        link[0..-3]
+      else
+        link
+      end
+    end
+
+    def self.get_page_links(page)
+      links = page.css("a:not([rel]),a[rel!=nofollow]").map do |link|
+        link.attribute("href").value rescue nil
+      end
+      links.uniq.compact
+    end
 end

data/lib/pageinfo/version.rb

@@ -1,3 +1,3 @@
 module Pageinfo
-  VERSION = "0.1.0"
+  VERSION = "0.1.9"
 end

data/pageinfo.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["adit@41studio.com"]
 
   spec.summary       = %q{Browse your page meta info and optimize your SEO strategy.}
-  spec.description   = %q{pageinfo will browse through your site then list every page on your site and give detail meta info about the page.}
+  spec.description   = %q{Pageinfo will browse through your site then list every page on your site and give detail meta info about the page.}
   spec.homepage      = "https://github.com/aditiamahdar/pageinfo"
   spec.license       = "MIT"
 
@@ -26,4 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.9"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"
+  spec.add_development_dependency "typhoeus", "~> 0.7"
+  spec.add_development_dependency "nokogiri", "~> 1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pageinfo
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.9
 platform: ruby
 authors:
 - aditiamahdar
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-11 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,11 +52,40 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
-description: pageinfo will browse through your site then list every page on your site
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: Pageinfo will browse through your site then list every page on your site
   and give detail meta info about the page.
 email:
 - adit@41studio.com
-executables: []
+executables:
+- pageinfo
 extensions: []
 extra_rdoc_files: []
 files:
@@ -68,7 +97,9 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/pageinfo
 - bin/setup
+- exe/pageinfo
 - lib/pageinfo.rb
 - lib/pageinfo/version.rb
 - pageinfo.gemspec