advent_of_ruby 0.2.0 → 0.3.0

data/lib/download_solutions/api/reddit.rb

@@ -0,0 +1,101 @@
+module DownloadSolutions
+  module Api
+    class Reddit
+      MaxSleepCountReachedError = Class.new(StandardError)
+      MultipleMoreChildrensError = Class.new(StandardError)
+
+      private attr_reader :user_agent, :client_id, :client_secret, :username, :password
+
+      # @param client_id [String]
+      # @param client_secret [String]
+      # @param username [String]
+      # @param password [String]
+      def initialize(client_id:, client_secret:, username:, password:)
+        @user_agent = "AdventOfRubyScript/#{Arb::VERSION} by fpsvogel"
+        @client_id = client_id
+        @client_secret = client_secret
+        @username = username
+        @password = password
+      end
+
+      # @param year [Integer]
+      # @param day [Integer]
+      # @param languages [Array<String>] e.g. ["ruby"]
+      # @return [Array<Hash>]
+      #
+      # @raise [MaxSleepCountReachedError] if Reddit seemingly throttled for an
+      #   unusually long time, "seemingly" because the only sign is an empty
+      #   JSON response body after loading additional comments.
+      # @raise [MultipleMoreChildrensError] if there are multiple "more children"
+      #   nodes for the thread or a comment; only one at a time is expected.
+      def get_comments(year:, day:, languages:)
+        params = Params.new(
+          year:,
+          day:,
+          languages:,
+          connection:
+        )
+
+        initial_response = GetInitialResponse.call(params:)
+        params.initial_response = initial_response
+
+        # Keep unfetched replies ("more children" nodes) separate so that after
+        # filtering, the replies to filtered-in comments can then be fetched.
+        original_comments, more_childrens = GetSerialComments.call(params:)
+        params.original_comments = original_comments
+        params.more_childrens = more_childrens
+
+        filtered_comments = FilterByLanguage.call(params:)
+        params.comments = filtered_comments
+
+        # These operations modify params#comments in place.
+        AddMissingReplies.call(params:)
+        RejectUnwantedReplies.call(params:)
+        CleanBodies.call(params:)
+        RemoveLanguageTags.call(params:)
+        RemoveIds.call(params:)
+
+        params.comments
+      end
+
+      private
+
+      def connection
+        @connection ||= Faraday.new(
+          url: "https://oauth.reddit.com",
+          headers: {
+            "User-Agent" => user_agent,
+            "Accept" => "application/json"
+          }
+        ) do |f|
+          f.request :authorization, "Bearer", -> { auth_token }
+          f.request :retry, {
+            max: 5,
+            interval: 0.5,
+            interval_randomness: 0.5,
+            backoff_factor: 2
+          }
+        end
+      end
+
+      def auth_token
+        connection = Faraday.new(
+          url: "https://www.reddit.com",
+          headers: {
+            "User-Agent" => user_agent
+          }
+        ) do |f|
+          f.request :authorization, :basic, client_id, client_secret
+          f.response :json
+        end
+
+        response = connection.post(
+          "/api/v1/access_token",
+          "grant_type=password&username=#{username}&password=#{password}"
+        )
+
+        response.body["access_token"]
+      end
+    end
+  end
+end

data/lib/download_solutions/cli/cli/shared.rb

@@ -0,0 +1,35 @@
+module DownloadSolutions
+  module Cli
+    private_class_method def self.validate_year_and_day(year:, day:)
+      if day
+        if year.nil?
+          raise InputError, "Year must be specified when day is specified."
+        end
+        if !day.between?(1, 25) && Date.new(year, 12, day) > Date.today
+          raise InputError, "Day must be between 1 and 25, and <= today."
+        end
+      end
+      if year && !year.between?(2015, Date.today.year)
+        raise InputError, "Year must be between 2015 and this year."
+      end
+    end
+
+    private_class_method def self.max_year_and_day(year:, day:)
+      if Date.today.year == year && Date.today.month == 12
+        [Date.today.year, Date.today.day]
+      else
+        [Date.today.year - 1, 25]
+      end
+    end
+
+    private_class_method def self.output_initial_message(source:, year:, day:, force:, detail:)
+      force_description = PASTEL.red("FORCE ") if force
+      year_description = year.nil? ? "all years" : year.to_s
+      day_description = day.nil? ? "" : "##{day.to_s.rjust(2, "0")}"
+      time_description = PASTEL.blue("#{year_description}#{day_description}")
+
+      puts "#{force_description}Downloading #{source} solutions from #{time_description} #{detail}..."
+      puts
+    end
+  end
+end

data/lib/download_solutions/cli/github.rb

@@ -0,0 +1,107 @@
+module DownloadSolutions
+  module Cli
+    def self.github(year: nil, day: nil, author: nil, force: false)
+      validate_year_and_day(year:, day:)
+
+      author_name = author.nil? ? "all authors" : PASTEL.blue(author)
+      detail = "by #{author_name}"
+      output_initial_message(source: "GitHub", year:, day:, force:, detail:)
+
+      github_directory = File.join("data", "solutions", "github")
+
+      repos = Api::Github::REPOS
+      if author
+        if repos.key?(author)
+          repos = repos.select { _1 == author }
+        else
+          raise InputError, "Repo author #{PASTEL.blue(author)} not found."
+        end
+      end
+
+      max_year, max_day = max_year_and_day(year:, day:)
+
+      authors = repos.keys
+      authors.each do |author|
+        author_directory = File.join(github_directory, author)
+        Dir.mkdir(author_directory) unless Dir.exist?(author_directory)
+
+        (year || 2015).upto(year || max_year) do |current_year|
+          year_directory = File.join(author_directory, current_year.to_s)
+
+          existing_solutions =
+            if Dir.exist?(year_directory)
+              Dir.entries(year_directory)
+                .filter_map {
+                  it.delete_suffix(".yml").split("_").map(&:to_i) if it.end_with?(".yml")
+                }.sort
+            else
+              []
+            end
+
+          solutions = github_api.get_solutions(
+            author:,
+            year: current_year,
+            input_day: day,
+            max_day:,
+            force:,
+            existing_solutions:
+          )
+
+          skipped_list = solutions_str(solutions[:skipped])
+          if solutions[:skipped].any?
+            puts "#{PASTEL.yellow.bold("Skipping")} from #{author} #{current_year}, already existing: #{PASTEL.yellow(skipped_list)}"
+          end
+
+          not_found_list = solutions_str(solutions[:not_found])
+          if solutions[:not_found].any?
+            puts "#{PASTEL.red.bold("Not found")} from #{author} #{current_year}: #{PASTEL.red(not_found_list)}"
+            if solutions[:not_found].size < 49
+              # Save empty files for not found solutions, so that they won't be
+              # attempted to be downloaded again.
+              Dir.mkdir(year_directory) unless Dir.exist?(year_directory)
+              solutions[:not_found].each do |(day, part)|
+                path = File.join(year_directory, "#{day.to_s.rjust(2, "0")}_#{part}.yml")
+                File.write(path, [].to_yaml)
+              end
+            end
+          end
+
+          if solutions[:new].any?
+            Dir.mkdir(year_directory) unless Dir.exist?(year_directory)
+            solutions[:new].each do |(day, part), content|
+              path = File.join(year_directory, "#{day.to_s.rjust(2, "0")}_#{part}.yml")
+              File.write(path, content.to_yaml(line_width: -1))
+            end
+
+            new_list = solutions_str(solutions[:new].keys)
+            puts "#{PASTEL.blue.bold("Saved")} from #{author} #{current_year}: #{PASTEL.blue(new_list)}"
+            puts "Saved to #{year_directory}"
+          end
+        end
+
+        puts
+      end
+    rescue InputError => e
+      puts PASTEL.red(e.message)
+    end
+
+    private_class_method def self.solutions_str(solutions)
+      if solutions.count == 49
+        "all"
+      else
+        solutions.sort.map { it.join(".") }.join(", ")
+      end
+    end
+
+    private_class_method def self.github_api
+      return @github_api if @github_api
+
+      github_token = "GITHUB_TOKEN"
+
+      Dotenv.load
+      Dotenv.require_keys([github_token])
+
+      @github_api = Api::Github.new(auth_token: ENV[github_token])
+    end
+  end
+end

data/lib/download_solutions/cli/reddit.rb

@@ -0,0 +1,64 @@
+module DownloadSolutions
+  module Cli
+    def self.reddit(year: nil, day: nil, languages: ["ruby"], force: false)
+      validate_year_and_day(year:, day:)
+
+      language_names = PASTEL.blue(languages.join(", "))
+      detail = "for #{language_names}"
+      output_initial_message(source: "Reddit", year:, day:, force:, detail:)
+
+      language_directory = File.join("data", "solutions", "reddit", languages.join("-"))
+      Dir.mkdir(language_directory) unless Dir.exist?(language_directory)
+
+      max_year, max_day = max_year_and_day(year:, day:)
+
+      (year || 2015).upto(year || max_year) do |current_year|
+        year_directory = File.join("data", "solutions", "reddit", languages.join("-"), current_year.to_s)
+        Dir.mkdir(year_directory) unless Dir.exist?(year_directory)
+
+        (day || 1).upto(day || max_day) do |current_day|
+          path = File.join("data", "solutions", "reddit", languages.join("-"), year.to_s, "#{current_day.to_s.rjust(2, "0")}.yml")
+
+          if File.exist?(path) && !force
+            puts PASTEL.yellow("Skipping #{PASTEL.yellow.bold("#{current_year}##{current_day.to_s.rjust(2, "0")}")} as it already exists.")
+            puts
+            next
+          end
+
+          comments = reddit_api.get_comments(
+            year: current_year,
+            day: current_day,
+            languages:
+          )
+
+          File.write(path, comments.to_yaml(line_width: -1))
+
+          puts "Saved #{PASTEL.blue("#{current_year}##{current_day.to_s.rjust(2, "0")}")} to #{path}"
+          puts
+        end
+      end
+    rescue InputError => e
+      puts PASTEL.red(e.message)
+    end
+
+    private_class_method def self.reddit_api
+      return @reddit_api if @reddit_api
+
+      reddit_api_keys = %w[
+        REDDIT_CLIENT_ID
+        REDDIT_CLIENT_SECRET
+        REDDIT_USERNAME
+        REDDIT_PASSWORD
+      ]
+
+      Dotenv.load
+      Dotenv.require_keys(reddit_api_keys)
+
+      reddit_api_kwargs = %i[client_id client_secret username password]
+        .zip(reddit_api_keys.map { ENV[it] })
+        .to_h
+
+      @reddit_api = Api::Reddit.new(**reddit_api_kwargs)
+    end
+  end
+end

data/lib/download_solutions/download_solutions.rb

@@ -0,0 +1,18 @@
+require "debug"
+require "dotenv"
+require "faraday"
+require "faraday/retry"
+require "pastel"
+require "redcarpet"
+require "reverse_markdown"
+require "yaml"
+require_relative "../arb/version"
+
+module DownloadSolutions
+  InputError = Class.new(StandardError)
+  PASTEL = Pastel.new
+end
+
+Dir[File.join(__dir__, "**", "*.rb")].each do |file|
+  require file
+end

data/lib/download_solutions/reverse_markdown/converters/br.rb

@@ -0,0 +1,15 @@
+# Monkeypatch. Reason: "  \n" is rendered as a line break by Markdown viewers,
+# but trailing spaces are annoying and I strip them out because that's what my
+# editor does on save anyway. Which leaves only the newline, which is rendered
+# as a space. So it's preferable for <br> to become a paragraph break.
+module ReverseMarkdown
+  module Converters
+    class Br < Base
+      def convert(node, state = {})
+        "\n\n" # CHANGED from "  \n"
+      end
+    end
+
+    register :br, Br.new
+  end
+end

data/lib/download_solutions/reverse_markdown/converters/pre.rb

@@ -0,0 +1,46 @@
+# Monkeypatch. Reason: leading spaces are significant for aligning lines of
+# code that are indented as well as wrapped in a fenced code block.
+module ReverseMarkdown
+  module Converters
+    class Pre < Base
+      def convert(node, state = {})
+        content = treat_children(node, state)
+        if ReverseMarkdown.config.github_flavored
+          # CHANGED from content.strip
+          "\n```#{language(node)}\n" << content.chomp << "\n```\n"
+        else
+          "\n\n    " << content.lines.to_a.join("    ") << "\n\n"
+        end
+      end
+
+      private
+
+      # Override #treat as proposed in https://github.com/xijo/reverse_markdown/pull/69
+      def treat(node, state)
+        case node.name
+        when "code", "text"
+          node.text.chomp # CHANGED from node.text.strip
+        when "br"
+          "\n"
+        else
+          super
+        end
+      end
+
+      def language(node)
+        lang = language_from_highlight_class(node)
+        lang || language_from_confluence_class(node)
+      end
+
+      def language_from_highlight_class(node)
+        node.parent["class"].to_s[/highlight-([a-zA-Z0-9]+)/, 1]
+      end
+
+      def language_from_confluence_class(node)
+        node["class"].to_s[/brush:\s?(:?.*);/, 1]
+      end
+    end
+
+    register :pre, Pre.new
+  end
+end