RedditPostClassifierBot 0.1.1

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/RedditPostClassifierBot.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'RedditPostClassifierBot/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "RedditPostClassifierBot"
+  spec.version       = RedditPostClassifierBot::VERSION
+  spec.authors       = ["Diego Salazar"]
+  spec.email         = ["diego@greyrobot.com"]
+
+  spec.summary       = %q{Run Naive Bayes classification of Reddit posts}
+  spec.description   = %q{This gem wraps Ruby's nbayes gem to do text classification of Reddit posts. It classifies posts according to where they were fetch - frontpage, controversial, top, or bad posts. It can be used to try to predict if a new post will make it to the front page, maybe.}
+  spec.homepage      = "https://github.com/DiegoSalazar/RedditPostClassifierBot"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+
+  spec.add_dependency "nbayes"
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "RedditPostClassifierBot"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/RedditPostClassifierBot/nbayes_classifier.rb

@@ -0,0 +1,34 @@
+module RedditPostClassifierBot
+  class NBayesClassifier
+    NBAYES_FILE = ENV.fetch "NBAYES_FILE_PATH", "./RPCB-nbayes.yml"
+
+    def initialize
+      @nbayes = NBayes::Base.new
+    end
+
+    def train(text, classification)
+      @nbayes.train tokenize(text), classification
+      self
+    end
+
+    def classify(subreddit, title, post)
+      @nbayes.classify tokenize("#{subreddit}\n#{title}\n#{post}")
+    end
+
+    def dump
+      @nbayes.dump NBAYES_FILE
+    end
+
+    def load
+      @nbayes.load NBAYES_FILE
+      true
+    rescue Errno::ENOENT, NoMethodError
+      FileUtils.touch NBAYES_FILE unless File.exists?(NBAYES_FILE)
+      false
+    end
+
+    private def tokenize(text = @text)
+      text.split(/\s+/)
+    end
+  end
+end

data/lib/RedditPostClassifierBot/reddit_trainer.rb

@@ -0,0 +1,121 @@
+module RedditPostClassifierBot
+  class RedditTrainer
+    REDDIT_URL = "https://www.reddit.com"
+    CLASSES = {
+      hot: "/",
+      new_post: "/new/",
+      rising: "/rising/",
+      controversial: "/controversial/",
+      top_hour: "/top/",
+      top_day: "/top/?sort=top&t=day",
+      top_week: "/top/?sort=top&t=week",
+      top_month: "/top/?sort=top&t=month",
+      top_year: "/top/?sort=top&t=year"
+    }
+
+    attr_reader :classifications
+
+    def self.trained_on
+      CLASSES.KEYS
+    end
+
+    def initialize(trials = 10, per_page = 200, debug = true)
+      @max_trials, @per_page, @debug = trials, per_page, debug
+      @posts, @trials_done = [], 0
+    end
+
+    def nbayes
+      @nbayes ||= RedditPostClassifierBot::NBayesClassifier.new
+    end
+
+    def train(classes = CLASSES)
+      classes.each do |classification, path|
+        log "training on #{classification} posts, page #{@trials_done} of #{@max_trials}"
+
+        reddit(path)["data"]["children"].each do |p|
+          @posts << (post = Post.new p)
+          nbayes.train post.serialize, classification
+        end
+      end
+
+      @trials_done += 1
+      recurse_to_next_page CLASSES, @posts.last if @trials_done <= @max_trials
+      self
+    end
+
+    def classify(subreddit, title, post)
+      @classifications = nbayes.classify subreddit, title, post
+      @classifications.max_class
+    end
+
+    def dump
+      nbayes.dump; self
+    end
+
+    def load
+      train and dump unless nbayes.load
+      self
+    end
+
+    def fetch_and_classify(path = CLASSES[:front])
+      posts = reddit(path)["data"]["children"]
+      log "Classifying #{posts.size} posts"
+
+      posts.inject({}) do |h, p|
+        post = Post.new p
+        classification = classify post.subreddit, post.title, post.body
+        h.merge! uri_with_base(post.path).to_s => classification
+      end.group_by { |_, c| c }
+    end
+
+    def inspect
+      "<#{self.class}:#{object_id.to_s(16)} @max_trials=#{@max_trials.inspect} @per_page=#{@per_page.inspect} @debug=#{@debug.inspect} @posts.size=#{@posts.size}>"
+    end
+
+    private 
+
+    def reddit(path)
+      path = path.split("?")
+      path = "#{path[0]}.json?#{path[1]}"
+      uri = uri_with_base path
+      cmd = "curl -s -c \"reddit_session=#{ENV["REDDIT_SESSION_ID"]}\" #{uri}"
+      
+      JSON.parse `#{cmd}` rescue { "data" => { "children" => [] }}
+    end
+
+    def recurse_to_next_page(classes, last_post)
+      classes = classes.inject({}) do |h, (k, v)|
+        u = uri_with_base v
+        u.query = "count=#{@per_page}&after=#{last_post.full_id}"
+        h.merge! k => u.to_s
+      end
+
+      train classes
+    end
+
+    def uri_with_base(path, base = REDDIT_URL)
+      URI.parse (s = URI.join(base, path)) rescue s # ignore bad uri
+    end
+
+    def log(msg)
+      puts "[#{self.class}] #{msg}" if @debug
+    end
+
+    class Post
+      attr_reader :full_id, :path, :subreddit, :title, :body
+
+      def initialize(json)
+        @data = json["data"]
+        @full_id = "#{json["kind"]}_#{@data["id"]}"
+        @path = @data["permalink"]
+        @subreddit = @data["subreddit"]
+        @title = @data["title"]
+        @body = (@data["selftext"].to_s.empty? ? @data["url"] : @data["selftext"]).to_s
+      end
+
+      def serialize
+        "#{@data["subreddit"]}\n#{@data["title"]}\n#{@data["url"]}\n#{@data["selftext"]}"
+      end
+    end
+  end
+end

data/lib/RedditPostClassifierBot/version.rb

@@ -0,0 +1,3 @@
+module RedditPostClassifierBot
+  VERSION = "0.1.1"
+end

data/lib/RedditPostClassifierBot.rb

@@ -0,0 +1,20 @@
+require "fileutils"
+require "nbayes"
+require "RedditPostClassifierBot/version"
+require "RedditPostClassifierBot/nbayes_classifier"
+require "RedditPostClassifierBot/reddit_trainer"
+require "RedditPostClassifierBot/good_post_finder"
+
+module RedditPostClassifierBot
+  def self.classifier
+    @classifier ||= RedditTrainer.new.load
+  end
+
+  def self.train_classifier
+    classifier.train.dump
+  end
+
+  def self.classify_posts(path)
+    classifier.fetch_and_classify path
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: RedditPostClassifierBot
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Diego Salazar
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nbayes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This gem wraps Ruby's nbayes gem to do text classification of Reddit
+  posts. It classifies posts according to where they were fetch - frontpage, controversial,
+  top, or bad posts. It can be used to try to predict if a new post will make it to
+  the front page, maybe.
+email:
+- diego@greyrobot.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- RPCB-nbayes.yml
+- Rakefile
+- RedditPostClassifierBot.gemspec
+- bin/console
+- bin/setup
+- lib/RedditPostClassifierBot.rb
+- lib/RedditPostClassifierBot/nbayes_classifier.rb
+- lib/RedditPostClassifierBot/reddit_trainer.rb
+- lib/RedditPostClassifierBot/version.rb
+homepage: https://github.com/DiegoSalazar/RedditPostClassifierBot
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Run Naive Bayes classification of Reddit posts
+test_files: []