aws-whitepaper-downloader 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1945c635d5d084efc5fa70451a474a9d4486b4de
+  data.tar.gz: f8dcd349cc8c967fd2f375103a463b4cea15ba96
+SHA512:
+  metadata.gz: 444f896f2c746ba7e44359c93bc79b1a043f66eac57c7052a6c590261c6cae0f9ce2ae6dcf56d4fc2e5cb2b21b6834e1ca6cc5aafdf0124c23c438f98e4fe15f
+  data.tar.gz: d069f3a3f9e522d2aa9db8d5e1de8bbcc5cff3f2bc1b569bedfa66162dd6818c631d87a3893e6cc7b3340984e909858a08430030afd9ae0f85a942a20af6494d

data/bin/aws-whitepaper-downloader

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/aws_whitepaper_downloader.rb'
+require "aws_whitepaper_downloader/downloader"
+require 'commander/import'
+require "fileutils"
+
+program :name, 'Aws White Paper Downloader'
+program :version, '1.0.0'
+program :description, 'download all whitepaper sorted by category'
+
+command :all do |c|
+  c.syntax = 'aws-whitepaper-downloader all'
+  c.description = ''
+  c.action do |args, options|
+    AwsWhitepaperDownloader.run( Dir.pwd )
+  end
+end

data/lib/aws_whitepaper_downloader.rb

@@ -0,0 +1,11 @@
+require "aws_whitepaper_downloader/version"
+require "aws_whitepaper_downloader/crawler"
+require "aws_whitepaper_downloader/downloader"
+require 'pry'
+
+module AwsWhitepaperDownloader
+  def self.run( dir_path=nil )
+    links_hash = Crawler.new.run
+    Downloader.new.run( links_hash, dir_path )
+  end
+end

data/lib/aws_whitepaper_downloader/crawler.rb

@@ -0,0 +1,78 @@
+require 'open-uri'
+require 'nokogiri'
+
+module AwsWhitepaperDownloader
+  class Crawler
+
+    def initialize
+    end
+
+    def run
+      count = 0
+      start = nil
+      output = {}
+      flag = 0
+      current_hash = nil
+      pdfs = nil
+      title = nil
+      doc = Nokogiri::HTML( open("https://aws.amazon.com/whitepapers/"))
+      doc.css(".content > .title-wrapper, .content > .row-builder").each do |div|
+
+        # div == title-wrapper
+        if div['class'].match("title-wrapper")
+          title = parse_title( div )
+          output[title] ||= {}
+          current_hash = output[title]
+        end
+
+        # div == row-builder
+        next unless current_hash
+        next unless div['class'].match("row-builder")
+        # nested
+        if div.css('h3').size > 0
+          current_hash['type'] = 'nested'
+          sub_divs = div.css(
+            ".wrapper > .columns > .parsys > .columnbuilder,
+            .wrapper > .columns > .parsys > .title-wrapper"
+          )
+          sub_divs.each do |sub_div|
+            if subtitle = parse_subtitle( sub_div )
+              current_hash[subtitle] ||= {}
+              pdfs = current_hash[subtitle]
+            end
+            pdfs = parse_pdfs(sub_div, pdfs)
+          end
+        # normal
+        else
+          current_hash['type'] = 'normal'
+          pdfs = output[title]
+          pdfs = parse_pdfs(div, pdfs)
+        end
+
+      end
+      return output
+    end
+
+    private
+
+    def parse_title( title_wrapper )
+      return title_wrapper.css('h1').text.strip
+    end
+
+    def parse_subtitle( row_builder )
+      h3_title = row_builder.css('h3').text.strip
+      return h3_title.size > 0 ? h3_title : nil
+    end
+
+    def parse_pdfs( html, pdfs )
+      html.css('li').each do |x|
+        pdf_name = x.css('b').text()
+        pdf_url = "https:#{ x.css('a').first['href'] }" if x.css('a').first
+        pdfs[pdf_name] = pdf_url if pdf_name.size > 0 && pdf_url.size > 0
+      end
+      return pdfs
+    end
+
+  end
+
+end

data/lib/aws_whitepaper_downloader/downloader.rb

@@ -0,0 +1,68 @@
+module AwsWhitepaperDownloader
+  class Downloader
+
+    WHITE_PAPPER_ROOT_DIR = "#{__dir__}/white pappers"
+
+    def initialize
+    end
+
+    def run( hash, download_path )
+      download_path = download_path ? "#{ download_path }/white pappers" : WHITE_PAPPER_ROOT_DIR
+      check_dir_created( nil, download_path )
+
+      hash.each_pair do |key, value|
+        current_dir = check_dir_created( key, download_path )
+        puts "---------------- current_dir -------------------"
+        puts current_dir
+        puts "-------------------------------------------------"
+        case value['type']
+        when 'normal'
+          Dir.chdir current_dir
+          puts "---------------- Dir.chdir curren_dir -------------------"
+          puts current_dir
+          puts "-------------------------------------------------"
+          puts 'normal'
+          hash.each_pair do |k1, v1|
+            next if k1 == 'type'
+            v1.each do |k2, v2|
+              next if k2 == 'type'
+              download( k2, v2,"#{current_dir}/#{v2}")
+            end
+          end
+        when 'nested'
+          puts 'nested'
+          value.each_pair do |k1, v1|
+            next if k1 == 'type'
+            nested_dir = check_dir_created( k1, current_dir )
+            Dir.chdir( nested_dir )
+            puts "---------------- nested_dir -------------------"
+            puts nested_dir
+            puts "------------ㄦ-----------------------------"
+
+            v1.each do |k2, v2|
+              next if k2 == 'type'
+              download( k2, v2,"#{nested_dir}/#{v2}")
+            end
+          end
+        end
+
+      end
+    end
+    # def run( download_dir )
+    # end
+
+    private
+
+    def download( file_name, path, download_dir )
+      unless File.exist? "#{file_name}.pdf"
+        `wget "#{file_name}.pdf" #{path}`
+      end
+    end
+    def check_dir_created( name, path )
+      location = "#{path}/#{name}"
+      Dir.mkdir location unless Dir.exist?( location )
+      return location
+    end
+
+  end
+end

data/lib/aws_whitepaper_downloader/version.rb

@@ -0,0 +1,3 @@
+module AwsWhitepaperDownloader
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: aws-whitepaper-downloader
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- StevenTTuD
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Downloading all aws whitepapers by 1 command
+email:
+- StevenTTuD@gmail.com
+executables:
+- aws-whitepaper-downloader
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/aws-whitepaper-downloader
+- lib/aws_whitepaper_downloader.rb
+- lib/aws_whitepaper_downloader/crawler.rb
+- lib/aws_whitepaper_downloader/downloader.rb
+- lib/aws_whitepaper_downloader/version.rb
+homepage: https://github.com/StevenTTuD/aws-whitepaper-downloader
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Downloading all aws whitepapers by 1 command
+test_files: []
+has_rdoc: