reddit_archiver 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 24a075b973c4926b6dde15536b3b2fa4a4cd20bc
+  data.tar.gz: 075052f1d7a6e61659bd80ce5cbf52f9900d720f
+SHA512:
+  metadata.gz: 31aaab845087793e3febfc3f23f214701fd4648d9873c23a961ac26e0cb7a2489bbd03eef927833ed1104e595c0ada4ba23d5d985f924ec7a685295f66e023a6
+  data.tar.gz: 6060034dfb3282c53472e1174c2038a704c5debf5fbcac7a9a95d5067cd182050846a2bdb2b41014a3f651af4f32d5b3faddb310beb2d1748b76a296fcb8420e

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+vendor/bundle
+config.yml
+*/comments
+*/submissions

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in reddit_archiver.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Timothy King
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# RedditArchiver
+
+**RedditArchiver** is a simple command line tool that will retrieve the contents of a reddit account, up to reddit's hard limit of 1000 items. This means that if your account contains a total of 1001 submissions or comments, only 1000 of them can be retrieved. There is currently no way around this limit.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'reddit_archiver'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install reddit_archiver
+
+## Usage
+
+    $ mv config.yml.example config.yml
+
+Edit the configuration file to supply your account details. Then, run
+
+    $ bundle exec reddit_archiver
+
+Once the script finishes, your posts will be available in the directory specified in config.yml.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/reddit_archiver

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby -Ilib
+
+require 'reddit_archiver'
+
+account = RedditArchiver::Account.new
+
+print "Retrieving posts:"
+posts = (account.submissions | account.comments)
+print " done with #{posts.count} items.\n"
+
+posts.each do |post|
+  print "Generating #{post.filename}:"
+  post.write
+  print " done.\n"
+end

data/config.yml.example

@@ -0,0 +1,3 @@
+username: myusername
+password: myawesomesecurepassword
+directory: defaults_to_your_username_if_ommitted

data/lib/reddit_archiver/account.rb

@@ -0,0 +1,50 @@
+require 'snoo'
+
+module RedditArchiver
+  class Account
+    attr_reader :connection, :config
+
+    def initialize
+      @config = RedditArchiver.config
+      @connection = Snoo::Client.new(username: config.username,
+                       password: config.password,
+                       useragent: "RedditArchiver #{RedditArchiver::VERSION}")
+    end
+
+    def comments
+      posts(:comments).map { |p| RedditArchiver::Comment.new(p) }
+    end
+
+    def submissions
+      posts(:submitted).map { |p| RedditArchiver::Submission.new(p) }
+    end
+
+    private
+
+    def posts(type)
+      # Reddit's API only allows a maximum of 100 comments at a time.
+      # To retrieve more than that (up to the hard limit of 1k), we
+      # have to get the 'name' attribute of the last comment and then
+      # request the next collection of comments that occur after it.
+      posts = retrieve_posts(type: type.to_s)
+      current_name = ""
+
+      begin
+        previous_name = current_name
+        current_name = posts.last["name"]
+        posts += retrieve_posts(type: type.to_s, after: current_name)
+      end until previous_name == current_name
+
+      posts
+    end
+
+    def retrieve_posts(args = {})
+      args.merge!(sort: 'new', limit: 100)
+      parse_response(connection.get_user_listing(connection.username, args))
+    end
+
+    def parse_response(http_response)
+      http_response["data"]["children"].map { |p| p["data"] }
+    end
+  end
+end

data/lib/reddit_archiver/comment.rb

@@ -0,0 +1,38 @@
+module RedditArchiver
+  class Comment < Post
+    attr_reader :body, :score, :link_title, :link_id, :link_author
+
+    def initialize(comment)
+      @body = comment["body"]
+      @score = comment["ups"] - comment["downs"]
+      @link_title = comment["link_title"]
+      @link_id = comment["link_id"]
+      @link_author = comment["link_author"]
+      super
+    end
+
+    def submission_link
+      "http://reddit.com/r/#{subreddit}/comments/#{link_id.split('_').last}"
+    end
+
+    def title
+      body[0..96]
+    end
+
+    def submission_title
+      link_title.parameterize[0..96]
+    end
+
+    def file_contents
+      <<-EOF.gsub(/^ {8}/, '')
+        Date: #{created}
+        On Post: #{link_title} by #{link_author}
+        Subreddit: /r/#{subreddit}
+        Link: #{submission_link}
+        Score: #{score}
+
+        #{body}
+      EOF
+    end
+  end
+end

data/lib/reddit_archiver/config.rb

@@ -0,0 +1,36 @@
+require 'yaml'
+
+module RedditArchiver
+  class Config
+    attr_reader :username, :password, :directory
+
+    def initialize(args = {})
+      @username, @password, @directory = YAML.load_file('config.yml').merge!(args).values_at("username", "password", "directory")
+      @directory ||= username
+
+      maybe_create_target_directory
+      maybe_create_target_subdirectories
+    end
+
+    def subdirectories
+      %w(submissions comments).map { |subdir| "#{directory}/#{subdir}" }
+    end
+
+    private
+
+    def maybe_create_target_directory
+      unless File.directory?(directory)
+        Dir.mkdir(directory)
+      end
+    end
+
+    def maybe_create_target_subdirectories
+      %w(submissions comments).each do |subdir|
+        target_subdir = File.join(directory, subdir)
+        unless File.directory?(target_subdir)
+          Dir.mkdir(target_subdir)
+        end
+      end
+    end
+  end
+end

data/lib/reddit_archiver/file_writer.rb

@@ -0,0 +1,35 @@
+module RedditArchiver
+  module FileWriter
+
+    def subdirectory
+      self.class.name.split('::').last.downcase.pluralize
+    end
+
+    def subdirectories
+      self.submission_title ? File.join(subdirectory, self.submission_title) : subdirectory
+    end
+
+    def filename
+      "#{self.year_month_date}_#{self.title_url}"[0..96]
+    end
+
+    def ext
+      ".md"
+    end
+
+    def filename_with_full_path
+      File.join(File.dirname(__FILE__), '../../', RedditArchiver.config.directory, subdirectories, filename + ext)
+    end
+
+    def target_directory
+      File.dirname(filename_with_full_path)
+    end
+
+    def write
+      Dir.mkdir(target_directory) unless File.directory?(target_directory)
+      file = File.new(filename_with_full_path, 'w')
+      file.puts(self.file_contents)
+      file.close
+    end
+  end
+end

data/lib/reddit_archiver/post.rb

@@ -0,0 +1,30 @@
+require 'htmlentities'
+require 'active_support/inflector'
+
+I18n.enforce_available_locales = false
+
+module RedditArchiver
+  class Post
+    include RedditArchiver::FileWriter
+
+    attr_reader :created, :subreddit, :reddit_id, :title
+
+    def initialize(post)
+      @created = Time.at(post["created"])
+      @subreddit = post["subreddit"]
+      @reddit_id = post["name"]
+    end
+
+    def title_url
+      title.parameterize
+    end
+
+    def submission_title
+      nil
+    end
+
+    def year_month_date
+      created.strftime('%F_%H-%M-%S')
+    end
+  end
+end

data/lib/reddit_archiver/submission.rb

@@ -0,0 +1,32 @@
+module RedditArchiver
+  class Submission < Post
+    attr_reader :permalink, :title, :url, :selftext, :score, :is_self
+
+    def initialize(submission)
+      @permalink = submission["permalink"]
+      @title = submission["title"]
+      @url = submission["url"]
+      @selftext = HTMLEntities.new.decode(submission["selftext"])
+      @score = submission["score"]
+      @is_self = submission["is_self"]
+      super
+    end
+
+    def body
+      is_self ? selftext : url
+    end
+
+    def file_contents
+      <<-EOF.gsub(/^ {8}/, '')
+        Date: #{created}
+        Subreddit: /r/#{subreddit}
+        Link: http://reddit.com#{permalink}
+        Score: #{score}
+
+        # #{title}
+
+        #{body}
+      EOF
+    end
+  end
+end

data/lib/reddit_archiver/version.rb

@@ -0,0 +1,3 @@
+module RedditArchiver
+  VERSION = "0.0.1"
+end

data/lib/reddit_archiver.rb

@@ -0,0 +1,13 @@
+require "reddit_archiver/version"
+require "reddit_archiver/config"
+require "reddit_archiver/account"
+require "reddit_archiver/file_writer"
+require "reddit_archiver/post"
+require "reddit_archiver/comment"
+require "reddit_archiver/submission"
+
+module RedditArchiver
+  def self.config
+    RedditArchiver::Config.new
+  end
+end

data/reddit_archiver.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'reddit_archiver/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "reddit_archiver"
+  spec.version       = RedditArchiver::VERSION
+  spec.authors       = ["Timothy King"]
+  spec.email         = ["tmk@lordzork.com"]
+  spec.description   = "Reddit_archiver is a command line tool that will download all of the posts and comments from a given reddit account and store them in flat text files."
+  spec.summary       = "A simple command line tool to archive the contents of a reddit account."
+  spec.homepage      = "http://github.com/tmking/reddit_archiver"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.1.1"
+  spec.add_development_dependency "rspec", "~> 2.14.1"
+  spec.add_development_dependency "webmock", "~> 1.17.1"
+  spec.add_runtime_dependency     "snoo", "~> 0.1.2"
+  spec.add_runtime_dependency     "activesupport", "~> 4.0.2"
+  spec.add_runtime_dependency     "htmlentities", "~> 4.3.1"
+end

data/spec/account_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe RedditArchiver::Account do
+  let(:connection) { double(Snoo::Client).as_null_object }
+  let(:parsed_response) {{ "json" => { "errors" => [] }}}
+  let(:http_response) { double("HTTP response", code: "200") }
+  let(:httparty_response) { double(HTTParty::Response,
+                               parsed_response: parsed_response,
+                               response: http_response) }
+
+  subject(:account) { RedditArchiver::Account.new }
+
+  before do
+    allow(Snoo::Client).to receive(:new).and_return(connection)
+  end
+
+  it "retreives the posts three times" do
+    expect(connection).to receive(:get_user_listing).exactly(3).times
+    account.comments
+  end
+end

data/spec/comment_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe RedditArchiver::Comment do
+  let(:post) { EXAMPLE_COMMENT["data"] }
+
+  subject { RedditArchiver::Comment.new(post) }
+
+  its(:year_month_date) { should eq("2013-09-19_00-29-37") }
+  its(:filename) { should eq('2013-09-19_00-29-37_hello') }
+  its(:body) { should eq('Hello.') }
+  its(:score) { should eq(3) }
+  its(:subdirectory) { should eq('comments') }
+  its(:title) { should eq('Hello.') }
+
+  context "The file" do
+    let(:comment) { RedditArchiver::Comment.new(post) }
+
+    before { comment.write }
+
+    it "is created" do
+      expect(File.exists?(comment.filename_with_full_path)).to be_true
+    end
+
+    it "has the contents of the post" do
+      file = File.read(comment.filename_with_full_path)
+      expect(file).to eq(comment.file_contents)
+    end
+  end
+end

data/spec/config_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe RedditArchiver::Config do
+  let(:yaml_data) { YAML.load(YAML_CONTENT) }
+
+  before { allow(YAML).to receive(:load_file).
+           with('config.yml').
+           and_return(yaml_data) }
+
+  context "With the values from the config file" do
+    subject { RedditArchiver::Config.new }
+    its(:username) { should eq('foo') }
+    its(:password) { should eq('bar') }
+    its(:directory) { should eq('spec/output/foobar') }
+  end
+
+  context "With no directory value" do
+    subject { RedditArchiver::Config.new }
+
+    before do
+      yaml_data.delete('directory')
+      Dir.stub(:mkdir).and_return(true)
+    end
+
+    its(:directory) { should eq('foo') }
+  end
+
+  context "Overriding the configuration values" do
+    subject { RedditArchiver::Config.new('username' => 'something',
+                                         'password' => 'else',
+                                         'directory' => 'altogether')
+    }
+
+    its(:username) { should eq('something') }
+    its(:password) { should eq('else') }
+    its(:directory) { should eq('altogether') }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'reddit_archiver'
+require 'webmock/rspec'
+WebMock.disable_net_connect!(allow_localhost: true)
+
+YAML_CONTENT = <<-EOF
+  username: foo
+  password: bar
+  directory: spec/output/foobar
+EOF
+
+PARSED_YAML = YAML.load(YAML_CONTENT)
+
+EXAMPLE_COMMENT = {"data" => {"subreddit_id"=>"t5_aaaaa", "link_title"=>"Test Test Test", "banned_by"=>nil, "subreddit"=>"test", "link_author"=>"test", "likes"=>true, "replies"=>nil, "saved"=>false, "id"=>"abc123", "gilded"=>0, "author"=>"me", "parent_id"=>"t3_aaaaa", "approved_by"=>nil, "body"=>"Hello.", "edited"=>false, "author_flair_css_class"=>"", "downs"=>1, "body_html"=>"&lt;div class=\"md\"&gt;&lt;p&gt;Hello.&lt;/p&gt;\n&lt;/div&gt;", "link_id"=>"t3_bbbbb", "score_hidden"=>false, "name"=>"t1_cccccc", "created"=>1379564977.0, "author_flair_text"=>"flair", "created_utc"=>1379561377.0, "ups"=>4, "num_reports"=>0, "distinguished"=>nil}}
+
+EXAMPLE_SUBMISSION = {"data" => {"subreddit_id"=>"t5_aaaaa", "link_title"=>"Test Test Test", "banned_by"=>nil, "subreddit"=>"test", "link_author"=>"test", "likes"=>true, "replies"=>nil, "saved"=>false, "id"=>"abc123", "gilded"=>0, "author"=>"me", "parent_id"=>"t3_aaaaa", "approved_by"=>nil, "body"=>"Hello.", "edited"=>false, "author_flair_css_class"=>"", "downs"=>1, "body_html"=>"&lt;div class=\"md\"&gt;&lt;p&gt;Hello.&lt;/p&gt;\n&lt;/div&gt;", "link_id"=>"t3_bbbbb", "score_hidden"=>false, "name"=>"t1_cccccc", "created"=>1379564977.0, "author_flair_text"=>"flair", "created_utc"=>1379561377.0, "ups"=>4, "num_reports"=>0, "distinguished"=>nil, "permalink" => "http://example.com", "title" => "Test post", "is_self" => true, "selftext" => "&gt; Hello.", "selftext_html" => "&nbsp;Hello.", "url" => "http://google.com"}}
+
+RSpec.configure do |config|
+  config.before(:each) do
+    YAML.stub(:load_file).and_call_original
+    YAML.stub(:load_file).with('config.yml').and_return(PARSED_YAML)
+  end
+  config.after(:all) { system('rm -Rf spec/output/*') }
+end
+

data/spec/submission_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe RedditArchiver::Submission do
+  let(:post) { EXAMPLE_SUBMISSION["data"] }
+  subject { RedditArchiver::Submission.new(post) }
+
+  its(:subdirectory) { should eq('submissions') }
+  its(:filename) { should eq('2013-09-19_00-29-37_test-post') }
+
+  context "When the post has its own text" do
+    its(:body) { should eq("> Hello.") }
+  end
+
+  context "When the post is just a link" do
+    before { post.merge!("is_self" => false) }
+    its(:body) { should eq("http://google.com") }
+  end
+
+  context "The file" do
+    let(:submission) { RedditArchiver::Submission.new(post) }
+
+    before { submission.write }
+
+    it "is created" do
+      expect(File.exists?(submission.filename_with_full_path)).to be_true
+    end
+
+    it "has the contents of the post" do
+      file = File.read(submission.filename_with_full_path)
+      expect(file).to eq(submission.file_contents)
+    end
+
+    it "truncates the name to 97 characters" do
+      long_filename = "f"*101
+      post = EXAMPLE_COMMENT["data"].merge!("title" => long_filename)
+      expect(RedditArchiver::Submission.new(post).filename.size).to eq(97)
+    end
+  end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: reddit_archiver
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Timothy King
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.1.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.1.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.17.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.17.1
+- !ruby/object:Gem::Dependency
+  name: snoo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.3.1
+description: Reddit_archiver is a command line tool that will download all of the
+  posts and comments from a given reddit account and store them in flat text files.
+email:
+- tmk@lordzork.com
+executables:
+- reddit_archiver
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/reddit_archiver
+- config.yml.example
+- lib/reddit_archiver.rb
+- lib/reddit_archiver/account.rb
+- lib/reddit_archiver/comment.rb
+- lib/reddit_archiver/config.rb
+- lib/reddit_archiver/file_writer.rb
+- lib/reddit_archiver/post.rb
+- lib/reddit_archiver/submission.rb
+- lib/reddit_archiver/version.rb
+- reddit_archiver.gemspec
+- spec/account_spec.rb
+- spec/comment_spec.rb
+- spec/config_spec.rb
+- spec/output/.keep
+- spec/spec_helper.rb
+- spec/submission_spec.rb
+homepage: http://github.com/tmking/reddit_archiver
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A simple command line tool to archive the contents of a reddit account.
+test_files:
+- spec/account_spec.rb
+- spec/comment_spec.rb
+- spec/config_spec.rb
+- spec/output/.keep
+- spec/spec_helper.rb
+- spec/submission_spec.rb