wpcrawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 319765da5c2b1b0f233b815b4d9165a6929934c376f40b8b9233b6544c357cd8
+  data.tar.gz: 8d6388658c421dd5064f849acc68e4825193799e544aa29124bb2be29e1c4cd6
+SHA512:
+  metadata.gz: c715ceaeee1b2763e20096551d463e9e2d660a7692c085a8fc7d28d41cce82fd8ad6c955731ffdb64c23bbce6a3b8469a259b528fd7ef42174d9f1f859c3d1a6
+  data.tar.gz: eacddd6bad4afd305dd708cd5809d23ec456139b8ba1d2b9d96d86657e6ba66d7872363145293faced26f7345def00378d77a9264e644dcc260d3fbf47558c7d

data/lib/wpcrawler.rb

@@ -0,0 +1,40 @@
+require 'JSON'
+require 'http'
+require 'tty-spinner'
+
+class Crawler
+  attr_accessor :url
+
+  def initialize(url, type)
+  	@url = url
+  	@type = type
+  end
+
+  def scrape
+  	begin
+  		spinner = TTY::Spinner.new("[:spinner]", format: :dots)
+  		spinner.auto_spin # Automatic animation with default interval
+	  	output = File.open( "wpoutputfile.csv","w+" )
+	    output.puts "Date | Title | Author | Link | Status"
+	    (1..8).each do |n|
+	      response = HTTP.timeout(5).get("https://" + @url.to_s + "/wp-json/wp/v2/#{@type}/" +"?page=#{n}&per_page=100").to_s
+	      JSON.parse(response).each do |e|
+	        output.puts "#{e.fetch('date')}" + "| #{e&.dig('title','rendered')}" + "| #{e.fetch('author')}"+ "| #{e.dig('link')}"+ "| #{e.dig('status')}"
+	      end rescue TypeError
+	    end
+	    output.close
+	    spinner.stop('Done!') # Stop animation
+	rescue HTTP::TimeoutError
+		puts "Connection ERROR - make sure your website is a wordpress site with an open api"
+	end
+  end
+end
+
+site = Crawler.new(ARGV[0], ARGV[1])
+
+if ARGV[1] == "pages" or ARGV[1] == "posts"
+  site.scrape
+else
+	puts "ERROR: type needs to be pages or posts"
+end
+

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: wpcrawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Duarte Martins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-22 00:00:00.000000000 Z
+dependencies: []
+description: A simple crawler that gets posts and pages from wordpress websites that
+  have an exposed api
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/wpcrawler.rb
+homepage: https://rubygems.org/gems/wpcrawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Simple wordpress crawler
+test_files: []