fourchan-kit 0.1.0

data/features/fourchan.feature

@@ -0,0 +1,24 @@
+Feature: Fourchan
+  In order to be useful
+  As a CLI
+  I want to be able to do stuff
+
+  Scenario: I need some help
+    When I run `fourchan help`
+    Then the output should contain "fourchan download"
+  
+  Scenario: I need some help to download
+    When I run `fourchan help download`
+    Then the output should contain "A valid URL for a thread"
+  
+  Scenario: I need some help to download
+    When I run `fourchan help lurk`
+    Then the output should contain "Where to save images"
+
+  @vcr
+  Scenario: I want to download a thread
+    When I run `fourchan download -u http://boards.4chan.org/g/thread/41705021`
+    Then the following folders should exist:
+      | tmp/aruba/images |
+    And the following folders should have "2" files combined:
+      | tmp/aruba/images |

data/features/step_definitions/cli_steps.rb

@@ -0,0 +1,11 @@
+Then /^the following folders should exist:$/ do |folders|
+  folders = folders.raw.flatten
+  folders.each { |f| File.exists?(f).should == true }
+end
+
+And /^the following folders should have "(\d+)" files combined:$/ do |count, folders|
+  folders = folders.raw.flatten
+  files   = 0
+  folders.each { |f| files += Dir["#{f}/**/*"].length }
+  files.should == count.to_i
+end

data/features/support/env.rb

@@ -0,0 +1,39 @@
+require 'aruba/cucumber'
+require 'aruba/in_process'
+require 'fourchan/kit/cli'
+require 'vcr'
+require 'webmock'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'features/cassettes'
+  c.hook_into :webmock
+end
+
+VCR.cucumber_tags do |t|
+  t.tag '@vcr', use_scenario_name: true 
+end
+
+# Magic by http://georgemcintosh.com/vcr-and-aruba/
+class VcrFriendlyMain
+  def initialize(argv, stdin, stdout, stderr, kernel)
+    @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+  end
+
+  def execute!
+    $stdin = @stdin
+    $stdout = @stdout
+    Fourchan::Kit::CLI.start(@argv)
+  end
+end
+
+Before('@vcr') do
+  Aruba::InProcess.main_class = VcrFriendlyMain
+  Aruba.process = Aruba::InProcess
+end
+
+After('@vcr') do
+  Aruba.process = Aruba::SpawnProcess
+  VCR.eject_cassette
+  $stdin = STDIN
+  $stdout = STDOUT
+end

data/fourchan-kit.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fourchan/kit/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fourchan-kit"
+  spec.version       = Fourchan::Kit::VERSION
+  spec.authors       = ["lauritzsh"]
+  spec.email         = ["mail@lauritz.me"]
+  spec.summary       = %q{A tool and API wrapper for the 4chan API.}
+  spec.description   = %q{Fourchan Kit is a Ruby wrapper and tool for the 4chan API. Use Fourchan Kit to interact with the API using Ruby, or use the tool to interact with the threads on 4chan.}
+  spec.homepage      = "http://rubygems.org/gems/fourchan-kit"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "json",      "~> 1.8"
+  spec.add_dependency "mechanize", "~> 2.7"
+  spec.add_dependency "thor",      "~> 0.19"
+
+  spec.add_development_dependency "aruba",    "~> 0.5"
+  spec.add_development_dependency "bundler",  "~> 1.6"
+  spec.add_development_dependency "cucumber", "~> 1.3"
+  spec.add_development_dependency "rake",     "~> 10.3"
+  spec.add_development_dependency "rspec",    "~> 2.14"
+  spec.add_development_dependency "vcr",      "~> 2.9"
+  spec.add_development_dependency "webmock",  "~> 1.17"
+end

data/lib/fourchan/kit.rb

@@ -0,0 +1,34 @@
+require "fourchan/kit/api"
+require "fourchan/kit/board"
+require "fourchan/kit/post"
+require "fourchan/kit/thread"
+require "fourchan/kit/tools"
+require "fourchan/kit/version"
+
+module Fourchan
+  module Kit
+    $fourchan_boards = []
+
+    ##
+    # Returns an array of all boards' title, such as _b_, _g_, _fit_ etc.
+    #
+    # @return [Array] name of all boards
+    def self.fourchan_boards
+      fetch_fourchan_boards if $fourchan_boards.empty?
+      $fourchan_boards
+    end
+
+    ##
+    # Updates the list of boards and returns an array of the boards.
+    #
+    # @return [Array] name of all boards
+    def self.fetch_fourchan_boards
+      puts "Fetching all 4chan boards"
+      $fourchan_boards = []
+      Fourchan::Kit::API.get_boards.each do |board|
+        $fourchan_boards << board["board"]
+      end
+      $fourchan_boards
+    end
+  end
+end

data/lib/fourchan/kit/api.rb

@@ -0,0 +1,62 @@
+require 'json'
+require 'open-uri'
+
+module Fourchan
+  module Kit
+
+    ##
+    # This module contains methods for the 4chan API.  
+    # They all parse the JSON 4chan delivers and returns a Hash object.
+    module API
+
+      ##
+      # Returns information for all boards across 4chan.
+      #
+      # @return [Array] information for all boards.
+      def self.get_boards
+        JSON.parse(open("http://a.4cdn.org/boards.json").read)['boards']
+      end
+
+      ##
+      # Returns information for all threads on specified board.
+      #
+      # @param board [String] the board.
+      # @return [Array] all threads for a board.
+      def self.get_catalog(board)
+        JSON.parse(open("http://a.4cdn.org/#{board}/catalog.json").read)
+      end
+
+      ##
+      # Returns only id and time for threads on specified board.
+      #
+      # @param board [String] the board.
+      # @return [Array] the id and time for all threads.
+      def self.get_threads(board)
+        JSON.parse(open("http://a.4cdn.org/#{board}/threads.json").read)
+      end
+
+      ##
+      # Returns all posts for the specified thread.
+      #
+      # @param board [String] the board.
+      # @param thread [Integer] the thread number.
+      # @return [Array] the posts in from a thread.
+      def self.get_thread(board, thread)
+        JSON.parse(open("http://a.4cdn.org/#{board}/thread/#{thread}.json").read)['posts']
+      end
+
+      ##
+      # Returns the threads at a page number on specified board.
+      #
+      # 4chan stopped using zero-index pages in April. Instead of first page
+      # is at 0, it is now at 1. 0 returns nothing.
+      #
+      # @param board [String] the board.
+      # @param page  [Integer] the thread number.
+      # @return [Array] all threads from a page.
+      def self.get_page(board, page)
+        JSON.parse(open("http://a.4cdn.org/#{board}/#{page}.json").read)['threads']
+      end
+    end
+  end
+end

data/lib/fourchan/kit/board.rb

@@ -0,0 +1,74 @@
+module Fourchan
+  module Kit
+
+    ##
+    # Board is used to deal with a 4chan board.
+    class Board
+      attr_reader :board
+
+      def initialize(board)
+        if Kit.fourchan_boards.include?(board)
+          @name  = board
+          @board = API.get_catalog(board)
+        else
+          raise "Not a valid board."
+        end
+      end
+
+      ##
+      # Returns only the first post (OP) from the threads on a page.
+      #
+      # @param page [Integer] the page to get threads from.
+      # @return [Array]
+      def threads(page = 1)
+        threads = []
+        @board[page - 1]["threads"].each do |thread|
+          threads << Post.new(thread, @name)
+        end
+        threads
+      end
+
+      ##
+      # Returns all threads, but not its replies, for the entire board.
+      #
+      # @return [Array]
+      def all_threads
+        all_threads = []
+        @board.each do |page|
+          all_threads << threads(page["page"])
+        end
+        all_threads.flatten
+      end
+
+      ##
+      # Returns all the posts from the threads on a page.
+      # 
+      # @param page [Integer] the page to get threads from.
+      # @return [Array]
+      def posts(page = 1)
+        posts   = []
+        threads = threads(page)
+        threads.each do |t|
+          thread = Thread.new(@name, t.no)
+          posts << thread.posts
+        end
+        posts.flatten
+      end
+
+      ##
+      # Returns all posts for the entire board.  
+      # *Note*: This method is pretty slow. Just wait for it to finish.
+      #
+      # @return [Array]
+      def all_posts
+        posts = []
+        @board.each_with_index do |_, i|
+          posts << posts(i + 1)
+        end
+        posts.flatten
+      end
+
+      alias_method :catalog, :board
+    end
+  end
+end

data/lib/fourchan/kit/cli.rb

@@ -0,0 +1,37 @@
+require 'fourchan/kit'
+require 'thor'
+
+module Fourchan
+  module Kit
+
+    class CLI < Thor
+      option :url,    aliases: '-u', desc: 'A valid URL for a thread'
+      option :file,   aliases: '-f', desc: 'Download images for every thread in a file'
+      option :out,    aliases: '-o', desc: 'In what folder should the images be saved to', default: 'images'
+      option :quiet,  aliases: '-q', desc: 'Do not output unecessary messages', type: :boolean
+      desc "download", "Download all images from a thread"
+      def download
+        url, file = options[:url], options[:file]
+        if url
+          Fourchan::Kit::Tools.download_thread(url, options.dup)
+        elsif file
+          Fourchan::Kit::Tools.download_threads(file, options.dup)
+        else
+          puts "I need some input to download the images. See `fourgem help download` for options."
+        end
+      end
+
+      option :timeout, aliases: '-t', desc: 'For how long should the thread be lurked. 0 to disable timeout', type: :numeric, default: 60
+      option :quiet,   aliases: '-q', desc: 'Do not output unecessary messages', type: :boolean
+      option :refresh, aliases: '-r', desc: 'How often to check for new replies', type: :numeric, default: 30
+      option :out,     aliases: '-o', desc: 'Where to save images', default: 'images'
+      # option :download, aliases: '-d', desc: 'Lurk for new images and download them', type: :boolean
+      # option :messages, aliases: '-m', desc: 'Lurk for new messages', type: :boolean, default: true
+      # option :out,      aliases: '-o', desc: 'There folder to store the images', default: 'images'
+      desc "lurk THREAD", "Look for new messages and/or download new images"
+      def lurk(thread)
+        options[:refresh] >= 5 ? Fourchan::Kit::Tools.lurk(thread, options.dup) : puts("Be fair, have refresh >= 5")
+      end
+    end
+  end
+end

data/lib/fourchan/kit/post.rb

@@ -0,0 +1,24 @@
+require 'ostruct'
+
+module Fourchan
+  module Kit
+
+    ##
+    # Post should make it easy to use OpenStruct for posts in a thread.  
+    # Also make it possible to get the link for the image, if the post has one.
+    class Post < OpenStruct
+      def initialize(hash, board)
+        super(hash)
+        @board = board
+      end
+
+      ##
+      # Return an URL for the image (if user submitted an image).
+      #
+      # @return [URL] the URL for the image.
+      def image_link
+        "http://i.4cdn.org/#{@board}/#{self.tim}#{self.ext}" if self.tim
+      end
+    end
+  end
+end

data/lib/fourchan/kit/thread.rb

@@ -0,0 +1,58 @@
+module Fourchan
+  module Kit
+
+    ##
+    # Thread is used to deal with a thread from a board.
+    class Thread
+      attr_reader :thread, :board
+
+      def initialize(board, thread)
+        @posts  = []
+        @board  = board
+        @thread = API.get_thread(board, thread)
+      end
+
+      ##
+      # Returns all posts from the thread, including OP.
+      def posts
+        if @posts.empty?
+          @thread.each do |post|
+            @posts << Post.new(post, @board)
+          end
+        end
+        @posts
+      end
+
+      ##
+      # Return only the first post from the thread.
+      def op
+        self.posts[0]
+      end
+
+      ##
+      # Get all replies from the thread. OP is not included.  
+      # It then returns the replies.
+      def fetch_replies
+        @posts = []
+        @thread = API.get_thread(@board, self.op.no)
+        self.replies
+      end
+
+      ##
+      # Return all the replies. OP is not included.
+      def replies
+        self.posts[1..-1]
+      end
+
+      ##
+      # Returns an array of image URLs from the thread (see {Fourgem::Post#image_link}).
+      def images
+        images = []
+        self.posts.each do |post|
+          images << post.image_link
+        end
+        images.compact
+      end
+    end
+  end
+end

data/lib/fourchan/kit/tools.rb

@@ -0,0 +1,158 @@
+require 'mechanize'
+require 'pathname'
+
+module Fourchan
+  module Kit
+
+    module Tools
+      $agent = Mechanize.new
+
+
+      ##
+      # Downloads the image from an URL.
+      #
+      # @param link [URL] the URL where the image is.
+      def self.download_image(link, options = {})
+        options[:fsize] ||= 0
+        options[:name]  ||= link.split('/').last
+        options[:out]   ||= "#{Dir.pwd}/images"
+        options[:quiet] ||= false
+
+        image = "#{create_dir(options[:out])}/#{options[:name]}"
+        unless File.exists?(image)
+          if valid_link?(link)
+            output = "Downloading: #{link}" unless options[:quiet]
+            output << (options[:fsize].zero? ? "" : " @ " << "#{(options[:fsize] / 1024.0).round(2)}kB".rjust(9))
+            puts output
+            $agent.get(link).save(image)
+          end
+        else
+          puts "Already got image, skipping" unless options[:quiet]
+        end
+      end
+
+      ##
+      # Downloads every image from a thread.
+      #
+      # @param link [URL] the URL for the thread to download.
+      def self.download_thread(link, options = {})
+        options[:checked] ||= false
+
+        if options[:checked] || ( valid_thread?(link) && valid_link?(link) )
+          board, thread_no = get_info(link)
+          thread = Thread.new(board, thread_no)
+
+          thread.posts.each do |post|
+            options[:fsize] = post.fsize
+            download_image(post.image_link, options.dup) if post.image_link
+          end
+        else
+          puts "Not a 4chan thread" unless options[:quiet]
+        end
+      end
+
+      ##
+      # Download all images from each thread in a file.
+      #
+      # Each thread must be on its own line and only be the URL, nothing else.  
+      # For example:
+      #     # threads.txt
+      #     http://boards.4chan.org/wg/thread/5777567
+      #     http://boards.4chan.org/wg/thread/5776602
+      #
+      # It takes care of dead threads or wrong URLs.
+      #
+      # @param file [File] the location of the file.
+      def self.download_threads(file, options = {})
+        options[:quiet] ||= false
+
+        if File.exists?(file)
+          File.open(file, 'r').each_line do |link|
+            puts "Getting images from thread: #{link}" unless options[:quiet]
+            if valid_thread?(link) && valid_link?(link)
+              options[:out]     = "images/#{link.scan(/(\d+)$/).first.first}"
+              options[:checked] = true
+              download_thread(link, options)
+              puts
+            else
+              puts "Not a 4chan thread" unless options[:quiet]
+              puts
+            end
+          end
+        else
+          puts "Not able to find the input file"
+        end
+      end
+
+      ##
+      # Check the thread for new images every x seconds.
+      #
+      # - The refresh rate is determined by options[:refresh] and is an integer.  
+      # - The time to lurk is determined by options[:timeout] and is an integer.
+      #
+      # @param link [URL] the thread to lurk
+      def self.lurk(link, options = {})
+        puts "Started lurking #{link}"
+
+        downloaded = []
+        board, thread_no = get_info(link)
+        thread = Thread.new(board, thread_no)
+
+        download_image(thread.op.image_link, options.dup)
+
+        begin
+          timeout(options[:timeout]) do
+            loop do
+              puts "Checking for images" unless options[:quiet]
+              new = thread.fetch_replies
+
+              (new - downloaded).each do |post|
+                options[:fsize] = post.fsize
+                download_image(post.image_link, options.dup) if post.image_link
+
+                downloaded << post
+              end
+
+              sleep(options[:refresh])
+            end
+          end
+        rescue Timeout::Error
+          puts "Timeout after #{options[:timeout]} second(s)"
+          exit 0
+        end
+      end
+
+      private
+      def self.create_dir(directory)
+        FileUtils.mkdir_p(directory) unless File.exists?(directory)
+        Pathname.new(directory).realpath.to_s
+      end
+
+      def self.get_info(link)
+        board  = link.scan(/(\w+)\/thread\//).first.first
+        thread = link.scan(/\/thread\/([0-9]+)/).first.first.to_i
+        [board, thread]
+      end
+
+      def self.valid_thread?(link)
+        link =~ /boards.4chan.org\/\w+\/thread\/\d+$/ ? true : false
+      end
+
+      def self.valid_link?(link)
+        begin
+          if link =~ /^#{URI::regexp(['http', 'https'])}$/
+            begin
+              $agent.get(link)
+            rescue Mechanize::ResponseCodeError
+              return false
+            end
+          else
+            return false
+          end
+
+          true
+        end
+      end
+    end
+  end
+end