mosquito-scrape 0.1.0

data/lib/mosquito/scrapers/scraper.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "capybara/dsl"
+require "dotenv/load"
+require "oj"
+require "selenium-webdriver"
+require "logger"
+require "securerandom"
+require "selenium/webdriver/remote/http/curb"
+require "debug"
+require "typhoeus"
+
+# 2022-06-07 14:15:23 WARN Selenium [DEPRECATION] [:browser_options] :options as a parameter for driver initialization is deprecated. Use :capabilities with an Array of value capabilities/options if necessary instead.
+
+options = Selenium::WebDriver::Options.chrome(exclude_switches: ["enable-automation"])
+options.add_argument("--start-maximized")
+options.add_argument("--no-sandbox")
+options.add_argument("--disable-dev-shm-usage")
+options.add_argument("–-disable-blink-features=AutomationControlled")
+options.add_argument("--disable-extensions")
+options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
+options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
+options.add_preference "password_manager_enabled", false
+options.add_argument("--user-data-dir=/tmp/tarun_zorki_#{SecureRandom.uuid}")
+
+Capybara.register_driver :selenium_mosquito do |app|
+  client = Selenium::WebDriver::Remote::Http::Curb.new
+  # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+  Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
+end
+
+Capybara.threadsafe = true
+Capybara.default_max_wait_time = 60
+Capybara.reuse_server = true
+
+module Mosquito
+  class Scraper # rubocop:disable Metrics/ClassLength
+    include Capybara::DSL
+
+    @@logger = Logger.new(STDOUT)
+    @@logger.level = Logger::WARN
+    @@logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+    @@session_id = nil
+
+    def initialize
+      Capybara.default_driver = :selenium_mosquito
+    end
+
+    def get_content_of_page_from_url_curl(url)
+      Typhoeus.get(url, followlocation: true)
+    end
+
+    # Instagram uses GraphQL (like most of Facebook I think), and returns an object that actually
+    # is used to seed the page. We can just parse this for most things.
+    #
+    # additional_search_params is a comma seperated keys
+    # example: `data,xdt_api__v1__media__shortcode__web_info,items`
+    #
+    # @returns Hash a ruby hash of the JSON data
+    def get_content_of_subpage_from_url(url, subpage_search, additional_search_parameters = nil)
+      # So this is fun:
+      # For pages marked as misinformation we have to use one method (interception of requrest) and
+      # for pages that are not, we can just pull the data straight from the page.
+      #
+      # How do we figure out which is which?... for now we'll just run through both and see where we
+      # go with it.
+
+      # Our user data no longer lives in the graphql object passed initially with the page.
+      # Instead it comes in as part of a subsequent call. This intercepts all calls, checks if it's
+      # the one we want, and then moves on.
+      response_body = nil
+
+      page.driver.browser.intercept do |request, &continue|
+        # This passes the request forward unmodified, since we only care about the response
+        # puts "checking request: #{request.url}"
+
+        continue.call(request) && next unless request.url.include?(subpage_search)
+
+
+        continue.call(request) do |response|
+          # Check if not a CORS prefetch and finish up if not
+          if !response.body.empty? && response.body
+            check_passed = true
+            unless additional_search_parameters.nil?
+              body_to_check = Oj.load(response.body)
+
+              search_parameters = additional_search_parameters.split(",")
+              search_parameters.each_with_index do |key, index|
+                break if body_to_check.nil?
+
+                check_passed = false unless body_to_check.has_key?(key)
+                body_to_check = body_to_check[key]
+              end
+            end
+
+            response_body = response.body if check_passed == true
+          end
+        end
+      rescue Selenium::WebDriver::Error::WebDriverError
+        # Eat them
+      end
+
+      # Now that the intercept is set up, we visit the page we want
+      page.driver.browser.navigate.to(url)
+      # We wait until the correct intercept is processed or we've waited 60 seconds
+      start_time = Time.now
+      # puts "Waiting.... #{url}"
+
+      sleep(rand(1...10))
+      while response_body.nil? && (Time.now - start_time) < 60
+        sleep(0.1)
+      end
+
+      page.driver.execute_script("window.stop();")
+      raise Mosquito::NoTweetFoundError if response_body.nil?
+      Oj.load(response_body)
+    end
+
+  private
+
+    ##########
+    # Set the session to use a new user folder in the options!
+    # #####################
+    def reset_selenium
+      options = Selenium::WebDriver::Options.chrome(exclude_switches: ["enable-automation"])
+      options.add_argument("--start-maximized")
+      options.add_argument("--no-sandbox")
+      options.add_argument("--disable-dev-shm-usage")
+      options.add_argument("–-disable-blink-features=AutomationControlled")
+      options.add_argument("--disable-extensions")
+      options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
+
+      options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 13_3_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
+      options.add_preference "password_manager_enabled", false
+      options.add_argument("--user-data-dir=/tmp/tarun_zorki_#{SecureRandom.uuid}")
+      # options.add_argument("--user-data-dir=/tmp/tarun")
+
+      Capybara.register_driver :selenium do |app|
+        client = Selenium::WebDriver::Remote::Http::Curb.new
+        # client.read_timeout = 60  # Don't wait 60 seconds to return Net::ReadTimeoutError. We'll retry through Hypatia after 10 seconds
+        Capybara::Selenium::Driver.new(app, browser: :chrome, options: options, http_client: client)
+      end
+
+      Capybara.current_driver = :selenium
+    end
+
+    def login
+      # Reset the sessions so that there's nothing laying around
+      page.quit
+
+      # Check if we're on a Instagram page already, if not visit it.
+      unless page.driver.browser.current_url.include? "instagram.com"
+        # There seems to be a bug in the Linux ARM64 version of chromedriver where this will properly
+        # navigate but then timeout, crashing it all up. So instead we check and raise the error when
+        # that then fails again.
+        page.driver.browser.navigate.to("https://instagram.com")
+      end
+
+      # We don't have to login if we already are
+      begin
+        return if find_field("Search", wait: 10).present?
+      rescue Capybara::ElementNotFound; end
+
+      # Check if we're redirected to a login page, if we aren't we're already logged in
+      return unless page.has_xpath?('//*[@id="loginForm"]/div/div[3]/button')
+
+      # Try to log in
+      loop_count = 0
+      while loop_count < 5 do
+        fill_in("username", with: ENV["INSTAGRAM_USER_NAME"])
+        fill_in("password", with: ENV["INSTAGRAM_PASSWORD"])
+
+        begin
+          click_button("Log in", exact_text: true) # Note: "Log in" (lowercase `in`) instead redirects to Facebook's login page
+        rescue Capybara::ElementNotFound; end # If we can't find it don't break horribly, just keep waiting
+
+        break unless has_css?('p[data-testid="login-error-message"', wait: 10)
+        loop_count += 1
+        sleep(rand * 10.3)
+      end
+
+      # Sometimes Instagram just... doesn't let you log in
+      raise "Instagram not accessible" if loop_count == 5
+
+      # No we don't want to save our login credentials
+      begin
+        click_on("Save Info")
+      rescue Capybara::ElementNotFound; end
+    end
+
+    def fetch_image(url)
+      request = Typhoeus::Request.new(url, followlocation: true)
+      request.on_complete do |response|
+        if request.success?
+          return request.body
+        elsif request.timed_out?
+          raise Zorki::Error("Fetching image at #{url} timed out")
+        else
+          raise Zorki::Error("Fetching image at #{url} returned non-successful HTTP server response #{request.code}")
+        end
+      end
+    end
+
+    # Convert a string to an integer
+    def number_string_to_integer(number_string)
+      # First we have to remove any commas in the number or else it all breaks
+      number_string = number_string.delete(",")
+      # Is the last digit not a number? If so, we're going to have to multiply it by some multiplier
+      should_expand = /[0-9]/.match(number_string[-1, 1]).nil?
+
+      # Get the last index and remove the letter at the end if we should expand
+      last_index = should_expand ? number_string.length - 1 : number_string.length
+      number = number_string[0, last_index].to_f
+      multiplier = 1
+      # Determine the multiplier depending on the letter indicated
+      case number_string[-1, 1]
+      when "m"
+        multiplier = 1_000_000
+      end
+
+      # Multiply everything and insure we get an integer back
+      (number * multiplier).to_i
+    end
+  end
+end
+
+# require_relative "tweet_scraper"

data/lib/mosquito/scrapers/tweet_scraper.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "typhoeus"
+require_relative "scraper"
+require "nokogiri"
+require "open-uri"
+
+module Mosquito
+  class TweetScraper < Scraper
+    def parse(id)
+      # Stuff we need to get from the DOM (implemented is starred):
+      # - User
+      # - Text *
+      # - Image  / Images  / Video *
+      # - Date *
+      # - Number of likes *
+      # - Hashtags
+
+      Capybara.app_host = ENV["NITTER_URL"]
+
+      # video slideshows https://www.instagram.com/p/CY7KxwYOFBS/?utm_source=ig_embed&utm_campaign=loading
+      # login
+      begin
+        doc = Nokogiri::HTML(URI.open("#{ENV["NITTER_URL"]}/jack/status/#{id}"))
+      rescue OpenURI::HTTPError
+        raise Mosquito::NoTweetFoundError
+      end
+
+      unless doc.xpath("//div[contains(@class, 'error-panel')]").empty?
+        raise Mosquito::NoTweetFoundError
+      end
+
+      text = doc.xpath("//div[contains(@class, 'tweet-content media-body')]").first.content
+      date = DateTime.parse(doc.xpath("//span[contains(@class, 'tweet-date')]").first.child["title"])
+      id = URI.parse(doc.xpath("//link[contains(@rel, 'canonical')]").first["href"]).path.split("/").last
+      number_of_likes = doc.xpath("//span[contains(@class, 'tweet-stat')][last()]/div").first.content.delete(",").to_i
+      language = "en" # We can't determine this anymore with Nitter, but english will be fine, we don't actually use this anywhere... i think
+      # user
+
+      images = []
+      videos = []
+      video_preview_image = nil
+      video_file_type = nil
+
+      # Single image
+      images.concat(doc.xpath("//a[contains(@class, 'still-image')][1]/href"))
+
+      # Slideshow
+      nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-row')]/div/a/@href")
+      images.concat(nodes.map { |node| "#{Capybara.app_host}#{node.value}" })
+
+      # Video
+      nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-video')]/div/video")
+      unless nodes.empty?
+        video_preview_image = nodes.first["poster"]
+        videos.concat(nodes.map { |node| "#{Capybara.app_host}#{node.xpath("//source").first["src"]}" })
+        video_file_type = "video" # This is always video now, sing a gif isn't displayed differently
+      end
+
+      # GIF
+      nodes = doc.xpath("//div[contains(@class, 'main-tweet')]/div/div/div[contains(@class, 'attachments')]/div[contains(@class, 'gallery-gif')]/div/video")
+      unless nodes.empty?
+        video_preview_image = nodes.first["poster"]
+        videos.concat(nodes.map { |node| "#{Capybara.app_host}#{node.xpath("//source[1]/source/@src").first&.content}" })
+        video_file_type = "gif"
+      end
+
+      username = doc.xpath("//a[contains(@class, 'username')][1]/@href").first.value
+      user = UserScraper.new.parse(username)
+
+      screenshot_file = take_screenshot()
+
+      {
+        images: images,
+        video: videos,
+        video_preview_image: video_preview_image,
+        screenshot_file: screenshot_file,
+        text: text,
+        date: date,
+        number_of_likes: number_of_likes,
+        user: user,
+        id: id,
+        language: language,
+        video_file_type: video_file_type
+      }
+    end
+
+    def take_screenshot
+      # First check if a post has a fact check overlay, if so, clear it.
+      # The only issue is that this can take *awhile* to search. Not sure what to do about that
+      # since it's Instagram's fault for having such a fucked up obfuscated hierarchy      # Take the screenshot and return it
+      save_screenshot("#{Mosquito.temp_storage_location}/instagram_screenshot_#{SecureRandom.uuid}.png")
+    end
+  end
+end

data/lib/mosquito/scrapers/user_scraper.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "typhoeus"
+require_relative "scraper"
+require "nokogiri"
+require "open-uri"
+
+module Mosquito
+  class UserScraper < Scraper
+    def parse(username)
+      # Stuff we need to get from the DOM (implemented is starred):
+      # id
+      # name
+      # username
+      # sign_up_date
+      # location
+      # profile_image_url
+      # description
+      # followers_count
+      # following_count
+      # tweet_count
+      # listed_count
+      # verified
+      # url
+      # profile_image_file_name
+
+      Capybara.app_host = ENV["NITTER_URL"]
+
+      username = username.delete("/")
+
+      doc = Nokogiri::HTML(URI.open("#{ENV["NITTER_URL"]}/#{username}"))
+
+      unless doc.xpath("//div[contains(@class, 'error-panel')]").empty?
+        raise Mosquito::NoTweetFoundError
+      end
+
+      id = username
+      full_name = doc.xpath("//a[contains(@class, 'profile-card-fullname')]/@title").first&.value
+      username = username
+      sign_up_date = DateTime.parse(doc.xpath("//div[contains(@class, 'profile-joindate')]/span/@title").first&.value)
+      location = doc.xpath("//div[contains(@class, 'profile-location')]/span[last()]").first&.content
+      profile_image_url = "#{Capybara.app_host}#{doc.xpath("//a[contains(@class, 'profile-card-avatar')]/@href").first&.value}"
+      description = doc.xpath("//div[contains(@class, 'profile-bio')]/p").first&.content
+      followers_count = doc.xpath("//li[contains(@class, 'followers')]/span[contains(@class, 'profile-stat-num')]").first&.content&.delete(",").to_i
+      following_count = doc.xpath("//li[contains(@class, 'following')]/span[contains(@class, 'profile-stat-num')]").first&.content&.delete(",").to_i
+      tweet_count = doc.xpath("//li[contains(@class, 'posts')]/span[contains(@class, 'profile-stat-num')]").first&.content&.delete(",").to_i
+      listed_count = 0 # We can't get this from nitter, and it's not a big deal
+      verified = !doc.xpath("//a[contains(@class, 'profile-card-fullname')]/div/span[contains(@title, 'Verified account')]").empty?
+      url = doc.xpath("//div[contains(@class, 'profile-website')]/span[last()]/a/@href").first&.content
+      profile_image_file_name = Mosquito.retrieve_media(profile_image_url)
+
+      user = {
+        id: id,
+        name: full_name,
+        username: username,
+        sign_up_date: sign_up_date,
+        location: location,
+        profile_image_url: profile_image_url,
+        description: description,
+        followers_count: followers_count,
+        following_count: following_count,
+        tweet_count: tweet_count,
+        listed_count: listed_count,
+        verified: verified,
+        url: url,
+        profile_image_file_name: profile_image_file_name
+      }
+
+      user
+    end
+
+    def take_screenshot
+      # First check if a post has a fact check overlay, if so, clear it.
+      # The only issue is that this can take *awhile* to search. Not sure what to do about that
+      # since it's Instagram's fault for having such a fucked up obfuscated hierarchy      # Take the screenshot and return it
+      save_screenshot("#{Mosquito.temp_storage_location}/instagram_screenshot_#{SecureRandom.uuid}.png")
+    end
+  end
+end

data/lib/mosquito/tweet.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Mosquito
+  class Tweet
+    def self.lookup(ids = [])
+      # If a single id is passed in we make it the appropriate array
+      ids = [ids] unless ids.kind_of?(Array)
+
+      # Check that the ids are at least real ids
+      ids.each { |id| raise Mosquito::InvalidIdError if !/\A\d+\z/.match(id) }
+
+      tweet_objects = ids.map { |id| Mosquito::TweetScraper.new.parse(id) }
+
+      tweet_objects.map do |tweet_object|
+        Tweet.new(tweet_object)
+      end
+    end
+
+    # Attributes for after the response is parsed from Twitter
+    attr_reader :json
+    attr_reader :id
+    attr_reader :created_at
+    attr_reader :text
+    attr_reader :language
+    attr_reader :author_id
+    attr_reader :author
+    attr_reader :image_file_names
+    attr_reader :video_file_names
+    attr_reader :video_file_type
+    attr_reader :video_preview_image
+
+    alias_method :user, :author # Every other gem uses `user` so we can just alias it
+
+  private
+
+    def initialize(tweet_object)
+      @id = tweet_object[:id]
+      @created_at = tweet_object[:date]
+      @text = tweet_object[:text]
+      @language = tweet_object[:language]
+      @author_id = tweet_object[:user][:id]
+
+      @image_file_names = tweet_object[:images]
+      @video_file_names = tweet_object[:video]
+      @video_file_type = tweet_object[:video_file_type]
+      @video_preview_image = tweet_object[:video_preview_image]
+      # Look up the author given the new id.
+      # NOTE: This doesn't *seem* like the right place for this, but I"m not sure where else
+      @author = User.new(tweet_object[:user])
+    end
+  end
+end

data/lib/mosquito/user.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Mosquito
+  class User
+    # Attributes for after the response is parsed from Twitter
+    attr_reader :json
+    attr_reader :id
+    attr_reader :name
+    attr_reader :username
+    attr_reader :sign_up_date
+    attr_reader :location
+    attr_reader :profile_image_url
+    attr_reader :description
+    attr_reader :url
+    attr_reader :followers_count
+    attr_reader :following_count
+    attr_reader :tweet_count
+    attr_reader :listed_count
+    attr_reader :verified
+    attr_reader :created_at
+    attr_reader :profile_image_file_name
+
+  private
+
+    def self.lookup(ids = [])
+      # If a single id is passed in we make it the appropriate array
+      ids = [ids] unless ids.kind_of?(Array)
+
+      # Check that the ids are at least real ids
+      user_objects = ids.map { |id| Mosquito::UserScraper.new.parse(id) }
+
+      user_objects.map do |user_object|
+        User.new(user_object)
+      end
+    end
+
+    def initialize(user_object)
+      @id = user_object[:id]
+      @name = user_object[:name]
+      @username = user_object[:username]
+      @created_at = user_object[:sign_up_date]
+      @location = user_object[:location]
+
+      # Removing the "normal" here gets us the full-sized image, instead of the 150x150 thumbnail
+      @profile_image_url = user_object[:profile_image_url]
+
+      @description = user_object[:description]
+      @url = user_object[:url]
+      @followers_count = user_object[:followers_count]
+      @following_count = user_object[:following_count]
+      @tweet_count = user_object[:tweet_count]
+      @listed_count = user_object[:listed_count]
+      @verified = user_object[:verified] # this will always be `false` but we're keeping it here for compatibility
+      @profile_image_file_name = user_object[:profile_image_file_name]
+    end
+  end
+end

data/lib/mosquito/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mosquito
+  VERSION = "0.1.0"
+end

data/lib/mosquito.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "json"
+require "typhoeus"
+require "date"
+require "securerandom"
+require "helpers/configuration"
+require "fileutils"
+
+require_relative "mosquito/version"
+require_relative "mosquito/tweet"
+require_relative "mosquito/user"
+require_relative "mosquito/scrapers/scraper"
+require_relative "mosquito/scrapers/tweet_scraper"
+require_relative "mosquito/scrapers/user_scraper"
+
+require_relative "mosquito/monkeypatch"
+
+module Mosquito
+  extend Configuration
+
+  class Error < StandardError; end
+  class AuthorizationError < Error; end
+  class InvalidIdError < Error; end
+  class InvalidMediaTypeError < Error; end
+  class NoTweetFoundError < Error; end
+  class RateLimitExceeded < Error
+    attr_reader :rate_limit
+    attr_reader :rate_remaining
+    attr_reader :reset_time_left
+
+    def initialize(rate_limit, rate_remaining, reset_time)
+    end
+  end
+
+  define_setting :temp_storage_location, "tmp/mosquito"
+  define_setting :nitter_url, ENV["NITTER_URL"]
+  define_setting :save_media, true
+
+  # The general fields to always return for Users
+  def self.user_fields
+    "name,created_at,location,profile_image_url,protected,public_metrics,url,username,verified,withheld,description"
+  end
+
+  # The general fields to always return for Tweets
+  def self.tweet_fields
+    "attachments,author_id,conversation_id,created_at,entities,geo,id,in_reply_to_user_id,lang"
+  end
+
+  # Get media from a URL and save to a temp folder set in the configuration under
+  # temp_storage_location
+  def self.retrieve_media(url)
+    return "" if !Mosquito.save_media
+
+    response = Typhoeus.get(url)
+
+    # Get the file extension if it's in the file
+    extension = url.split(".").last
+
+    # Do some basic checks so we just empty out if there's something weird in the file extension
+    # that could do some harm.
+    if extension.length.positive?
+      extension = extension[0...extension.index("?")]
+      extension = nil unless /^[a-zA-Z0-9]+$/.match?(extension)
+      extension = ".#{extension}" unless extension.nil?
+    end
+
+    temp_file_name = "#{Mosquito.temp_storage_location}/#{SecureRandom.uuid}#{extension}"
+
+    # We do this in case the folder isn't created yet, since it's a temp folder we'll just do so
+    self.create_temp_storage_location
+    File.binwrite(temp_file_name, response.body)
+    temp_file_name
+  end
+
+private
+
+  def self.create_temp_storage_location
+    return if File.exist?(Mosquito.temp_storage_location) && File.directory?(Mosquito.temp_storage_location)
+    FileUtils.mkdir_p Mosquito.temp_storage_location
+  end
+end

data/mosquito.gemspec

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "lib/mosquito/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "mosquito-scrape"
+  spec.version       = Mosquito::VERSION
+  spec.authors       = ["Christopher Guess"]
+  spec.email         = ["cguess@gmail.com"]
+
+  spec.summary       = "A gem to scrape a Nitter instance for Twitter data"
+  # spec.description   = "TODO: Write a longer description or delete this line."
+  # spec.homepage      = "TODO: Put your gem's website or public repo URL here."
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  # spec.metadata["homepage_uri"] = spec.homepage
+  # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Prod dependencies
+  spec.add_dependency "typhoeus", "~> 1.4"
+  spec.add_dependency "nokogiri", "~> 1.15.5"
+  spec.add_dependency "capybara", "~> 3.39"
+  spec.add_dependency "dotenv", "~> 2.8"
+  spec.add_dependency "oj", "~> 3.16"
+  spec.add_dependency "fileutils", "~> 1.7"
+  spec.add_dependency "logger", "~> 1.6"
+  spec.add_dependency "securerandom", "~> 0.3"
+  spec.add_dependency "selenium-webdriver", "~> 4"
+  spec.add_dependency "open-uri", "~> 0.4"
+  spec.add_dependency "activesupport", "~> 7.0.8"
+  spec.add_dependency "rack", "~> 2"
+
+  # Dev dependencies
+  spec.add_development_dependency "byebug", "~> 11.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "minitest", "~> 5"
+  spec.add_development_dependency "rubocop", "~> 1.0"
+  spec.add_development_dependency "rubocop-rails", "~> 2.0"
+  spec.add_development_dependency "rubocop-rails_config", "~> 1.0"
+  spec.add_development_dependency "rubocop-performance", "~> 1.0"
+  spec.add_development_dependency "dotenv", "~> 2.0"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end