pointmd_comments 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b3f84ad7c180458bae9b8d07589647dae11825d3580bbbec79602fb22cafcb8a
+  data.tar.gz: 060d0ce0518fb48dbd105ca9c3084e687ced21d08c8e1ee1458e8a47eea8f7d7
+SHA512:
+  metadata.gz: 77a227fd2f38df75955f865350451a8a905bdeaa2a7ed959822c8c0322499923e4e20ab1ea17e1fe19a1c974190d565150617c288a29dca5bc9326bb645bb686
+  data.tar.gz: b8584558d781b6cd09c566d3e2c6b1455686f40efe22edb7745867f6ece79de6b13f2b7ae6719010efb1bb22785e8070ebd703545d43bb12b588ab4ddf1a4b45

data/bin/pointmd_comments

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'pointmd_comments'
+
+PointmdComments.collect

data/lib/pointmd_comments.rb

@@ -0,0 +1,31 @@
+require 'nokogiri'
+require 'open-uri'
+require 'csv'
+require 'watir'
+
+require 'pointmd_comments/version'
+require 'pointmd_comments/opt_parser'
+
+require 'pointmd_comments/aggregators/main'
+require 'pointmd_comments/aggregators/posts'
+require 'pointmd_comments/aggregators/comments'
+
+require 'pointmd_comments/errors/not_implemented'
+require 'pointmd_comments/errors/unknown_source'
+
+require 'pry'
+
+module PointmdComments
+  class Error < StandardError; end
+
+  def self.collect
+    # DOESN'T WORK!!
+    args = ARGV.dup
+    puts args
+    options = OptParser.new.parse
+    Aggregators::Main.new(options).call
+  rescue StandardError => e
+    puts e.class, e.message
+    puts e.backtrace if args.include? '-v'
+  end
+end

data/lib/pointmd_comments/aggregators/comments.rb

@@ -0,0 +1,21 @@
+module PointmdComments
+  module Aggregators
+    class Comments
+      def call(page)
+        parse_comments(page)
+      end
+
+      private
+
+      def parse_comments(page)
+        comments = page.css('div#simpals-comments-list').children.first.children
+        comments.map do |comment|
+          comment.css('p').children.last.text
+
+        rescue StandardError
+          next
+        end.compact
+      end
+    end
+  end
+end

data/lib/pointmd_comments/aggregators/main.rb

@@ -0,0 +1,53 @@
+module PointmdComments
+  module Aggregators
+    class Main
+      attr_reader :posts_aggregator, :browser, :comments_aggregator, :all_comments, :posts, :source, :path
+
+      def initialize(options)
+        @path                = options[:path]
+        @source              = options[:source]
+        @posts_aggregator    = Aggregators::Posts.new(source: source, path: path)
+        @comments_aggregator = Aggregators::Comments.new
+        @browser             = ::Watir::Browser.new :chrome, headless: true
+        @all_comments        = []
+      end
+
+      def call
+        @posts = posts_aggregator.call
+
+        collect_all_comments
+
+        write_to_csv
+      end
+
+      private
+
+      def collect_all_comments
+        posts.each do |url|
+          post_comments = collect_comments_from(url)
+
+          post_comments.each do |c|
+            all_comments << ["https://point.md#{url}", c]
+          end
+        end
+      end
+
+      def collect_comments_from(url)
+        puts "Collecting comments for #{url}.."
+
+        browser.goto "https://point.md#{url}"
+        browser.div(id: 'simpals-comments-list').wait_until(&:present?)
+
+        page = Nokogiri::HTML.parse(browser.html)
+
+        comments_aggregator.call(page)
+      end
+
+      def write_to_csv
+        CSV.open('output.csv', 'w') do |csv|
+          all_comments.each { |c| csv << c }
+        end
+      end
+    end
+  end
+end

data/lib/pointmd_comments/aggregators/posts.rb

@@ -0,0 +1,67 @@
+module PointmdComments
+  module Aggregators
+    class Posts
+      # NOTE: This array may be populated with other website sections in the future.
+      ALLOWED_SOURCES = %i[news today].freeze
+      MAIN_PAGE       = 'https://point.md/ru/'.freeze
+
+      attr_reader :source, :urls
+
+      def initialize(source:, path:)
+        @source = source
+        @path   = path
+        @urls   = []
+      end
+
+      def call
+        validate_source
+        fetch_posts
+        puts "Found #{urls.count} links in the #{source} section.."
+
+        @urls
+      end
+
+      private
+
+      def validate_source
+        return if ALLOWED_SOURCES.include? source
+
+        raise ArgumentError, "Wrong source. Allowed sources are #{ALLOWED_SOURCES}"
+      end
+
+      def fetch_posts
+        @page = download_html
+        case source
+        when :news
+          fetch_news_posts
+        when :today
+          raise Errors::NotImplemented
+        end
+      end
+
+      def fetch_news_posts
+        posts_block = @page.css('.post-blocks-wrap')
+        main_post_heading = posts_block.children.css('.post-big-block').children.css('h2')
+
+        main_post   = main_post_heading.children.css('a').first['href']
+        other_posts = posts_block.children.css('.post-small-blocks-wrap')
+
+        populate_urls(main_post, other_posts)
+      end
+
+      def populate_urls(main_post, other_posts)
+        @urls << main_post
+
+        @urls += other_posts.children.map do |child|
+          child.css('article').css('h2').children[1]['href']
+        rescue StandardError
+          next
+        end.compact
+      end
+
+      def download_html
+        Nokogiri::HTML(open('https://point.md/ru'))
+      end
+    end
+  end
+end

data/lib/pointmd_comments/errors/not_implemented.rb

@@ -0,0 +1,5 @@
+module PointmdComments
+  module Errors
+    class NotImplemented < StandardError; end
+  end
+end

data/lib/pointmd_comments/errors/unknown_source.rb

@@ -0,0 +1,13 @@
+module PointmdComments
+  module Errors
+    class UnknownSource < StandardError
+      attr_reader :message
+
+      def initialize
+        super
+
+        @message = "Unknown source specified. Allowed sources are: #{Aggregators::Posts::ALLOWED_SOURCES}"
+      end
+    end
+  end
+end

data/lib/pointmd_comments/opt_parser.rb

@@ -0,0 +1,60 @@
+require 'optparse'
+require 'pointmd_comments/version'
+
+module PointmdComments
+  class OptParser
+    attr_reader :options
+
+    def initialize
+      @options = {}
+    end
+
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def parse
+      OptionParser.new do |opts|
+        opts.banner = 'Usage: pointmd_comments [options]'
+
+        opts.on('-v', '--verbose', 'Show logs and backtraces') do |v|
+          options[:verbose] = v
+        end
+
+        opts.on(
+          '-sSOURCE',
+          '--source=SOURCE',
+          "A source to pull links from. Available sources are: #{Aggregators::Posts::ALLOWED_SOURCES}"
+        ) do |s|
+          options[:source] = s.to_sym
+        end
+        opts.on(
+          '-p PATH',
+          '--path=PATH',
+          'A file path to pull links from.'
+        ) do |s|
+          options[:source] = s.to_sym
+        end
+        opts.on('-V', '--version', 'Version') do
+          puts PointmdComments::VERSION
+          exit
+        end
+      end.parse!
+
+      set_default_source unless options[:source] && options[:path]
+      validate_source
+
+      options
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+    private
+
+    def set_default_source
+      options[:source] = :news
+    end
+
+    def validate_source
+      return if Aggregators::Posts::ALLOWED_SOURCES.include? options[:source]
+
+      raise Errors::UnknownSource
+    end
+  end
+end

data/lib/pointmd_comments/version.rb

@@ -0,0 +1,3 @@
+module PointmdComments
+  VERSION = '0.1.0'.freeze
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: pointmd_comments
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nicolai Stoianov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: watir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: watir-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem lets you aggregate comments from point.md website into a csv
+  file.
+email:
+- stoianovnk@gmail.com
+executables:
+- pointmd_comments
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/pointmd_comments
+- lib/pointmd_comments.rb
+- lib/pointmd_comments/aggregators/comments.rb
+- lib/pointmd_comments/aggregators/main.rb
+- lib/pointmd_comments/aggregators/posts.rb
+- lib/pointmd_comments/errors/not_implemented.rb
+- lib/pointmd_comments/errors/unknown_source.rb
+- lib/pointmd_comments/opt_parser.rb
+- lib/pointmd_comments/version.rb
+homepage: https://github.com/ston1x/pointmd-comments
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/ston1x/pointmd-comments
+  source_code_uri: https://github.com/ston1x/pointmd-comments
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: Point.md comments aggregator
+test_files: []