phishtank_scraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0688aee54e521f9a6a96fd3e48b7f7b53dce9c7d
+  data.tar.gz: 9c2918a1aee0a431e6d7d941fe0bce6ccc2048f9
+SHA512:
+  metadata.gz: 129c7e9814004ed43b57ec64dfb6481dfa5081414903e3942e3639fa26b7ae885abd1fd754313db27184716a3e48da89e4619fc716b37a4478a5327c7a551666
+  data.tar.gz: d50292dcb0d1b6e657d1b69743af3c2335631521de0403af93c48825cb60a1fae6205d45a7c766c0d48452ba8e2f5034956583e7bd3a4ad15b0e270e01dc5b97

data/lib/phishtank_scraper/phishing_set.rb

@@ -0,0 +1,72 @@
+require 'nokogiri'
+# require 'random_user_agent'
+require 'open-uri'
+
+# Creates a collection of phishing hashes for each phishtank page
+# Example
+# {
+#   id: "4141251",
+#   url: "http://bintango.xyz/AIsaE",
+#   created_at: "added on Jun 3rd 2016 3:57 PM",
+#   submitter: "PhishReporter",
+#   valid: "Unknown",
+#   online: "ONLINE"
+# }
+
+class	PhishingSet
+  include Enumerable
+
+  attr_reader :url, :all
+
+  def initialize(url)
+    @url = URI(url)
+
+    @page = Nokogiri::HTML(open(@url.to_s))
+    @all = scrape_parse
+  end
+
+  def scrape_parse
+    rows = @page.at('.data').search('tr') 
+    rows.shift #removes header
+
+    rows.collect do |row|
+      id = row.at_xpath('td[1]/a/text()').to_s.strip
+      url_id = row.at_xpath('td[1]/a/@href').to_s.strip
+
+      url = row.at_xpath('td[2]/text()').to_s.strip
+      url = scrape_detail(url_id) if url[-3,3] == "..." # incomplete url
+
+      {
+        id: id,
+        url: url,
+        created_at: row.at_xpath('td[2]/span/text()').to_s.strip,
+        submitter: row.at_xpath('td[3]/a/text()').to_s.strip,
+        valid: row.at_xpath('td[4]/text()').to_s.strip,
+        online: row.at_xpath('td[5]/strong/text()').to_s.strip
+      }
+    end
+  end
+
+  def scrape_detail(url)
+
+    uri = URI(url)
+    uri = uri.host ? uri : "#{@url.scheme}://#{@url.host}/#{uri}"
+
+    detail_page = Nokogiri::HTML(open(uri))
+    detail_page.at("#widecol").at_xpath("div/div[3]/b/text()").to_s
+  end
+
+  def first
+    @all.first
+  end
+
+  def page_at_id(id)
+    last_subm_id = self.first[:id].to_i
+    ((last_subm_id - id + 1)/20).round # 20 items per page
+  end
+
+  def each
+    @all.each{ |ph| yield ph }
+  end
+
+end

data/lib/phishtank_scraper/site.rb

@@ -0,0 +1,32 @@
+# Prvides routes and URLs
+class Site
+  attr_reader :domain
+
+  def initialize(url)
+    @domain = url
+  end
+
+  def home
+    build_path
+  end
+  
+  def build_path(page_index=0, options={})
+    active = options[:active] 
+    valid = options[:valid] 
+
+    path = if active or valid
+      actives =  "&active=" + (active || "y")
+      valid =  "&valid=" + (valid || "y")
+
+      "phish_search.php?page=#{page_index}#{active}#{valid}&Search=Search"
+    else
+      "phish_archive.php?page=#{page_index}"
+    end
+
+    "#{@domain}/#{path}"
+  end
+ 
+  def build_detail_path(submission_id)
+    "#{@domain}/phish_detail.php?phish_id=#{submission_id}"
+  end
+end

data/lib/phishtank_scraper.rb

@@ -0,0 +1,42 @@
+require 'phishtank_scraper/phishing_set'
+require 'phishtank_scraper/site'
+
+# Director interface for scraping
+class PhishtankScraper
+  attr_reader :site, :range
+
+  def initialize(url="http://phishtank.com")
+    @site = Site.new(url)
+    @range = (0..0)
+  end
+
+  # returns an array of detections in the pages range 
+  # options: 
+  # active: "All", "n", "y", "u"
+  # valid: "All", "n", "y", "u"
+  def page_scrape(range=@range, options={})
+    build_range(range).map do |page_index|
+      PhishingSet.new(@site.build_path(page_index, options)).all
+    end.flatten
+  end
+
+  # returns an array of detections from id to last submitted id
+  # options: 
+  # active: "All", "n", "y", "u"
+  # valid: "All", "n", "y", "u"
+  def id_scrape(since, options={})
+    since = since.to_i
+    page_at = PhishingSet.new(@site.home).page_at_id(since)
+
+    phset = (0..page_at).map do |page_index|
+      PhishingSet.new(@site.build_path(page_index, options)).all
+    end.flatten
+    
+    phset.delete_if {|ph| ph[:id].to_i < since}
+  end
+
+  private
+  def build_range(value)
+    @range = value.class.eql?(Range) ? value : (value..value)
+  end 
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: phishtank_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Marlon Méndez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.7.2
+description: Scrapes Phishtank submissions given a page interval or an id
+email: marlonmendezg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/phishtank_scraper.rb
+- lib/phishtank_scraper/phishing_set.rb
+- lib/phishtank_scraper/site.rb
+homepage: http://rubygems.org/gems/phishtank_scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Scrapes Phishtank submissions
+test_files: []