ginspider 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29fa6c15ccb053fa0634bbfa9507de6a1d5a63a4
-  data.tar.gz: ba3dbf3cbade0a9234e52f81ed5c3b553f25175d
+  metadata.gz: 24178b5f71bc5c092364a9066975e62e453cebd8
+  data.tar.gz: 6bfd4f30b2ac4146f3c99ff056991be04d3a02e9
 SHA512:
-  metadata.gz: 8562aa005f132860a2038670e505edb00ce2021c33eb041dff6f25ad4a5ae730468583669bf70a68449398e9a57f32b1272c1c18fe717f7753291007f7de3503
-  data.tar.gz: 143b860949224be710f653d373f4fb2c64af49875421dc9123afe64d9a3cbe2c04772ec6220f592a3d776d4cfdafaca369302a6aa4cc21abf7e97b2abf56a7e6
+  metadata.gz: 18be42fc8f3ef9cf67231039228aa42d0a47998396a8b65da21b943b32436ef847b29721d32d11eb2730ca11d6e6b09ecc36420188fcbcd575e5c8420a6c0cea
+  data.tar.gz: a36c97cbb54a850be74b2a569ce2ca55e268cf28339f82a263361ab94a65d2b198d3fd5c4cc26b52b4a8e3b56b1e1320cee775bc0e201ea44bfe8557bf828013

data/README.md

@@ -1,4 +1,5 @@
 # Ginspider
+[![Build Status](https://travis-ci.org/LanceGin/GinSpider.svg?branch=master)](https://travis-ci.org/LanceGin/GinSpider)
 
 Ginspider is a powerful gem to analyze the common websites, it will be easier to get the HTTP request data.
 

data/lib/ginspider/analysis.rb

@@ -9,18 +9,53 @@ module Ginspider
     # return nokogiri response
     # 
     def self.noko_res(url)
-      res_body = Http.res_body(url)
-      @noko_res = get_noko_res(res_body)
-      return @noko_res
+      res_body = get_res_body(url)
+      noko_res = get_noko_res(res_body)
+      return noko_res
+    end
+
+    # 
+    # return links of response with url and options
+    # 
+    def self.res_links(url, options = {})
+      res_body = get_res_body(url)
+      res_links = get_res_links(res_body, options)
+      return res_links
     end
     
     private
+
+    # 
+    # get the response body
+    # 
+    def self.get_res_body(url)
+      @res_body = Http.res_body(url)
+      return @res_body
+    end
+
     # 
     # generate a nokogiri response
     # 
     def self.get_noko_res(res_body)
-      noko_res = Nokogiri::HTML(res_body.to_s)
-      return noko_res
+      @noko_res = Nokogiri::HTML(res_body.to_s)
+      return @noko_res
+    end
+
+    # 
+    # get all the links from nokogiri object
+    # 
+    def self.get_res_links(res_body, options = {})
+      @noko_res = get_noko_res(res_body)
+      @links = @noko_res.css(options["element"])
+
+      links = []
+      i = 0
+      @links.each do |link|
+        links[i] = link["href"].strip
+        i = i + 1
+      end
+
+      return links
     end
 
   end

data/lib/ginspider/main.rb

@@ -5,21 +5,39 @@ require 'ginspider/analysis'
 
 module Ginspider
 
-  # 
-  # the main method to crawl a website
-  # 
-  def Ginspider.crawl(url, options = {})
-    Main.crawl(url, options)
-  end
-
   class Main
     
     # 
-    # define the main crawler
+    # get the basic response
     # 
     def self.crawl(url, options = {})
+      basic_res = basic_res(url)
+      ap basic_res
+    end
+
+    # define the main crawler
+    # 
+    def self.crawl_with_noko(url, options = {})
       noko_res = Analysis.noko_res(url)
-      ap noko_res.css(options["element"]).text
+      ap noko_res
+    end
+
+    # 
+    # define the main method to get all links
+    # 
+    def self.links_of(url, options = {})
+      res_links = Analysis.res_links(url, options)
+      ap res_links
+    end
+
+    private
+
+    # 
+    # get the basic response
+    # 
+    def self.basic_res(url, options = {})
+      @basic_res = Http.basic_res(url)
+      return @basic_res
     end
 
   end

data/lib/ginspider/version.rb

@@ -1,3 +1,3 @@
 module Ginspider
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/ginspider.rb

@@ -1,2 +1,27 @@
 require 'rubygems'
-require 'ginspider/main'
+require 'ginspider/main'
+
+module Ginspider
+	
+  # 
+  # the api method to crawl a website, and get the basic response
+  # 
+  def Ginspider.crawl(url, options = {})
+    Main.crawl(url, options)
+  end
+
+  # 
+  # the api method to get all links with url and options
+  # 
+  def Ginspider.links_of(url, options = {})
+  	Main.links_of(url, options)
+  end
+
+  # 
+  # the api method to get text with url and options
+  # 
+  def Ginspider.text_of(url, options = {})
+  	Main.text_of(url, options)
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ginspider
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - lancegin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-08 00:00:00.000000000 Z
+date: 2016-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: to analyze some websites