twitter_ebooks 2.3.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c11c99337f28c765e0061e6e9a541ae3def2eb3
-  data.tar.gz: aeb8bf213b41972ea1257e3b5f4bba14a2df88c3
+  metadata.gz: 5207fc0b43e38b2aeefa7f995be02cbbdf98f1d2
+  data.tar.gz: 79dd79c4cfeee86ff9bb20e790f19473156cfbba
 SHA512:
-  metadata.gz: 07955ae0c6fb60549ad3fb48dc742ea6378e889c9ef9300bce7a78ef56ab11990c1a2d76f5046f62b1b75dca9dbb5165860d70d5db3dc6a7e5be28915a306f19
-  data.tar.gz: 3d4a3459c7fe881c92773fe3d0b40dc32e440aaa208cf95f010a8d541fa826760a86370adcbbda1a4f93e448ace3d3089fd454eb808c37cdbbcd0bdbd870dc04
+  metadata.gz: da424019fae0c0b9383be7fcc1feb80a562f3612c6991f49eeda4dd3864c905ce6e3c0497b85ed001ec2eac7d81d2b92a0fd1271bad8f6c539ad017852e8e11f
+  data.tar.gz: e332b40106b7dd9df79704e80ee67fe4798458c8cb483f21e1302dd99f3cfa5a1f6dcf3327654e605c13f42884abc67f3eccd30f4475c7a62e2d7ea882bc894f

data/.gitignore

@@ -1,3 +1,5 @@
 .*.swp
 Gemfile.lock
 pkg
+.yardoc
+doc

data/.travis.yml

@@ -0,0 +1,7 @@
+rvm:
+  - 2.1.4
+script:
+  - rspec spec
+notifications:
+  email:
+    - ebooks@mispy.me

data/README.md

@@ -1,10 +1,26 @@
-# twitter\_ebooks 2.3.2
+# twitter\_ebooks
 
-Rewrite of my twitter\_ebooks code. While the original was solely a tweeting Markov generator, this framework helps you build any kind of interactive twitterbot which responds to mentions/DMs. See [ebooks\_example](https://github.com/mispy/ebooks_example) for an example of a full bot.
+[![Gem Version](https://badge.fury.io/rb/twitter_ebooks.svg)](http://badge.fury.io/rb/twitter_ebooks)
+[![Build Status](https://travis-ci.org/mispy/twitter_ebooks.svg)](https://travis-ci.org/mispy/twitter_ebooks)
+[![Dependency Status](https://gemnasium.com/mispy/twitter_ebooks.svg)](https://gemnasium.com/mispy/twitter_ebooks)
+
+A framework for building interactive twitterbots which respond to mentions/DMs. See [ebooks_example](https://github.com/mispy/ebooks_example) for a fully-fledged bot definition.
+
+## New in 3.0
+
+- Bots run in their own threads (no eventmachine), and startup is parallelized
+- Bots start with `ebooks start`, and no longer die on unhandled exceptions
+- `ebooks auth` command will create new access tokens, for running multiple bots
+- `ebooks console` starts a ruby interpreter with bots loaded (see Ebooks::Bot.all)
+- Replies are slightly rate-limited to prevent infinite bot convos
+- Non-participating users in a mention chain will be dropped after a few tweets
+- [API documentation](http://rdoc.info/github/mispy/twitter_ebooks)
+
+Note that 3.0 is not backwards compatible with 2.x, so upgrade carefully!
 
 ## Installation
 
-Requires Ruby 1.9.3+ (2.1+ recommended)
+Requires Ruby 2.0+
 
 ```bash
 gem install twitter_ebooks
@@ -16,48 +32,63 @@ Run `ebooks new <reponame>` to generate a new repository containing a sample bot
 
 ``` ruby
 # This is an example bot definition with event handlers commented out
-# You can define as many of these as you like; they will run simultaneously
+# You can define and instantiate as many bots as you like
+
+class MyBot < Ebooks::Bot
+  # Configuration here applies to all MyBots
+  def configure
+    # Consumer details come from registering an app at https://dev.twitter.com/
+    # Once you have consumer details, use "ebooks auth" for new access tokens
+    self.consumer_key = '' # Your app consumer key
+    self.consumer_secret = '' # Your app consumer secret
 
-Ebooks::Bot.new("abby_ebooks") do |bot|
-  # Consumer details come from registering an app at https://dev.twitter.com/
-  # OAuth details can be fetched with https://github.com/marcel/twurl
-  bot.consumer_key = "" # Your app consumer key
-  bot.consumer_secret = "" # Your app consumer secret
-  bot.oauth_token = "" # Token connecting the app to this account
-  bot.oauth_token_secret = "" # Secret connecting the app to this account
+    # Users to block instead of interacting with
+    self.blacklist = ['tnietzschequote']
 
-  bot.on_message do |dm|
+    # Range in seconds to randomize delay when bot.delay is called
+    self.delay_range = 1..6
+  end
+
+  def on_startup
+    scheduler.every '24h' do
+      # Tweet something every 24 hours
+      # See https://github.com/jmettraux/rufus-scheduler
+      # bot.tweet("hi")
+      # bot.pictweet("hi", "cuteselfie.jpg")
+    end
+  end
+
+  def on_message(dm)
     # Reply to a DM
     # bot.reply(dm, "secret secrets")
   end
 
-  bot.on_follow do |user|
+  def on_follow(user)
     # Follow a user back
     # bot.follow(user[:screen_name])
   end
 
-  bot.on_mention do |tweet, meta|
+  def on_mention(tweet)
     # Reply to a mention
-    # bot.reply(tweet, meta[:reply_prefix] + "oh hullo")
+    # bot.reply(tweet, meta(tweet)[:reply_prefix] + "oh hullo")
   end
 
-  bot.on_timeline do |tweet, meta|
+  def on_timeline(tweet)
     # Reply to a tweet in the bot's timeline
-    # bot.reply(tweet, meta[:reply_prefix] + "nice tweet")
+    # bot.reply(tweet, meta(tweet)[:reply_prefix] + "nice tweet")
   end
+end
 
-  bot.scheduler.every '24h' do
-    # Tweet something every 24 hours
-    # See https://github.com/jmettraux/rufus-scheduler
-    # bot.tweet("hi")
-	# bot.pictweet("hi", "cuteselfie.jpg", ":possibly_sensitive => true")
-  end
+# Make a MyBot and attach it to an account
+MyBot.new("{{BOT_NAME}}") do |bot|
+  bot.access_token = "" # Token connecting the app to this account
+  bot.access_token_secret = "" # Secret connecting the app to this account
 end
 ```
 
-Bots defined like this can be spawned by executing `run.rb` in the same directory, and will operate together in a single eventmachine loop. The easiest way to run bots in a semi-permanent fashion is with [Heroku](https://www.heroku.com); just make an app, push the bot repository to it, enable a worker process in the web interface and it ought to chug along merrily forever.
+'ebooks start' will run all defined bots in their own threads. The easiest way to run bots in a semi-permanent fashion is with [Heroku](https://www.heroku.com); just make an app, push the bot repository to it, enable a worker process in the web interface and it ought to chug along merrily forever.
 
-The underlying [tweetstream](https://github.com/tweetstream/tweetstream) and [twitter gem](https://github.com/sferik/twitter) client objects can be accessed at `bot.stream` and `bot.twitter` respectively.
+The underlying streaming and REST clients from the [twitter gem](https://github.com/sferik/twitter) can be accessed at `bot.stream` and `bot.twitter` respectively.
 
 ## Archiving accounts
 
@@ -92,7 +123,6 @@ Text files use newlines and full stops to seperate statements.
 Once you have a model, the primary use is to produce statements and related responses to input, using a pseudo-Markov generator:
 
 ``` ruby
-> require 'twitter_ebooks'
 > model = Ebooks::Model.load("model/0xabad1dea.model")
 > model.make_statement(140)
 => "My Terrible Netbook may be the kind of person who buys Starbucks, but this Rackspace vuln is pretty straight up a backdoor"
@@ -103,14 +133,14 @@ Once you have a model, the primary use is to produce statements and related resp
 The secondary function is the "interesting keywords" list. For example, I use this to determine whether a bot wants to fav/retweet/reply to something in its timeline:
 
 ``` ruby
-top100 = model.keywords.top(100)
+top100 = model.keywords.take(100)
 tokens = Ebooks::NLP.tokenize(tweet[:text])
 
 if tokens.find { |t| top100.include?(t) }
-  bot.twitter.favorite(tweet[:id])
+  bot.favorite(tweet[:id])
 end
 ```
 
-## Other notes
+## Bot niceness
 
-If you're using Heroku, which has no persistent filesystem, automating the process of archiving, consuming and updating can be tricky. My current solution is just a daily cron job which commits and pushes for me, which is pretty hacky.
+twitter_ebooks will drop bystanders from mentions for you and avoid infinite bot conversations, but it won't prevent you from doing a lot of other spammy things. Make sure your bot is a good and polite citizen!

data/bin/ebooks

@@ -2,54 +2,85 @@
 # encoding: utf-8
 
 require 'twitter_ebooks'
-require 'csv'
+require 'ostruct'
 
-$debug = true
+module Ebooks::Util
+  def pretty_exception(e)
 
-module Ebooks
+  end
+end
+
+module Ebooks::CLI
   APP_PATH = Dir.pwd # XXX do some recursive thing instead
+  HELP = OpenStruct.new
 
-  def self.new(reponame)
-    usage = <<STR
-Usage: ebooks new <reponame>
+  HELP.default = <<STR
+Usage:
+     ebooks help <command>
 
-Creates a new skeleton repository defining a template bot in
-the current working directory specified by <reponame>.
+     ebooks new <reponame>
+     ebooks auth
+     ebooks consume <corpus_path> [corpus_path2] [...]
+     ebooks consume-all <corpus_path> [corpus_path2] [...]
+     ebooks gen <model_path> [input]
+     ebooks archive <username> [path]
+     ebooks tweet <model_path> <botname>
 STR
 
+  def self.help(command=nil)
+    if command.nil?
+      log HELP.default
+    else
+      log HELP[command].gsub(/^ {4}/, '')
+    end
+  end
+
+  HELP.new = <<-STR
+    Usage: ebooks new <reponame>
+
+    Creates a new skeleton repository defining a template bot in
+    the current working directory specified by <reponame>.
+  STR
+
+  def self.new(reponame)
     if reponame.nil?
-      log usage
-      exit
+      help :new
+      exit 1
     end
 
     path = "./#{reponame}"
 
     if File.exists?(path)
       log "#{path} already exists. Please remove if you want to recreate."
-      exit
+      exit 1
     end
 
-    FileUtils.cp_r(SKELETON_PATH, path)
+    FileUtils.cp_r(Ebooks::SKELETON_PATH, path)
 
     File.open(File.join(path, 'bots.rb'), 'w') do |f|
-      template = File.read(File.join(SKELETON_PATH, 'bots.rb'))
+      template = File.read(File.join(Ebooks::SKELETON_PATH, 'bots.rb'))
       f.write(template.gsub("{{BOT_NAME}}", reponame))
     end
 
+    File.open(File.join(path, 'Gemfile'), 'w') do |f|
+      template = File.read(File.join(Ebooks::SKELETON_PATH, 'Gemfile'))
+      f.write(template.gsub("{{RUBY_VERSION}}", RUBY_VERSION))
+    end
+
     log "New twitter_ebooks app created at #{reponame}"
   end
 
-  def self.consume(pathes)
-    usage = <<STR
-Usage: ebooks consume <corpus_path> [corpus_path2] [...]
+  HELP.consume = <<-STR
+    Usage: ebooks consume <corpus_path> [corpus_path2] [...]
 
-Processes some number of text files or json tweet corpuses
-into usable models. These will be output at model/<name>.model
-STR
+    Processes some number of text files or json tweet corpuses
+    into usable models. These will be output at model/<name>.model
+  STR
 
+  def self.consume(pathes)
     if pathes.empty?
-      log usage
-      exit
+      help :consume
+      exit 1
     end
 
     pathes.each do |path|
@@ -57,24 +88,43 @@ STR
       shortname = filename.split('.')[0..-2].join('.')
 
       outpath = File.join(APP_PATH, 'model', "#{shortname}.model")
-      Model.consume(path).save(outpath)
+      Ebooks::Model.consume(path).save(outpath)
       log "Corpus consumed to #{outpath}"
     end
   end
 
-  def self.gen(model_path, input)
-    usage = <<STR
-Usage: ebooks gen <model_path> [input]
+  HELP.consume_all = <<-STR
+    Usage: ebooks consume-all <name> <corpus_path> [corpus_path2] [...]
 
-Make a test tweet from the processed model at <model_path>.
-Will respond to input if provided.
-STR
+    Processes some number of text files or json tweet corpuses
+    into one usable model. It will be output at model/<name>.model
+  STR
+
+  def self.consume_all(name, paths)
+    if paths.empty?
+      help :consume_all
+      exit 1
+    end
+
+    outpath = File.join(APP_PATH, 'model', "#{name}.model")
+    Ebooks::Model.consume_all(paths).save(outpath)
+    log "Corpuses consumed to #{outpath}"
+  end
+
+  HELP.gen = <<-STR
+    Usage: ebooks gen <model_path> [input]
+
+    Make a test tweet from the processed model at <model_path>.
+    Will respond to input if provided.
+  STR
+
+  def self.gen(model_path, input)
     if model_path.nil?
-      log usage
-      exit
+      help :gen
+      exit 1
     end
 
-    model = Model.load(model_path)
+    model = Ebooks::Model.load(model_path)
     if input && !input.empty?
       puts "@cmd " + model.make_response(input, 135)
     else
@@ -82,136 +132,208 @@ STR
     end
   end
 
-  def self.score(model_path, input)
-    usage = <<STR
-Usage: ebooks score <model_path> <input>
-
-Scores "interest" in some text input according to how
-well unique keywords match the model.
-STR
-    if model_path.nil? || input.nil?
-      log usage
-      exit
-    end
+  HELP.archive = <<-STR
+    Usage: ebooks archive <username> [outpath]
 
-    model = Model.load(model_path)
-    model.score_interest(input)
-  end
+    Downloads a json corpus of the <username>'s tweets.
+    Output defaults to corpus/<username>.json
+    Due to API limitations, this can only receive up to ~3000 tweets
+    into the past.
+  STR
 
-  def self.archive(username, outpath)
-    usage = <<STR
-Usage: ebooks archive <username> <outpath>
-
-Downloads a json corpus of the <username>'s tweets to <outpath>.
-Due to API limitations, this can only receive up to ~3000 tweets
-into the past.
-STR
-
-    if username.nil? || outpath.nil?
-      log usage
-      exit
+  def self.archive(username, outpath=nil)
+    if username.nil?
+      help :archive
+      exit 1
     end
 
-    Archive.new(username, outpath).sync
+    Ebooks::Archive.new(username, outpath).sync
   end
 
-  def self.tweet(modelpath, botname)
-    usage = <<STR
-Usage: ebooks tweet <model_path> <botname>
+  HELP.tweet = <<-STR
+    Usage: ebooks tweet <model_path> <botname>
 
-Sends a public tweet from the specified bot using text
-from the processed model at <model_path>.
-STR
+    Sends a public tweet from the specified bot using text
+    from the processed model at <model_path>.
+  STR
 
+  def self.tweet(modelpath, botname)
     if modelpath.nil? || botname.nil?
-      log usage
-      exit
+      help :tweet
+      exit 1
     end
 
     load File.join(APP_PATH, 'bots.rb')
-    model = Model.load(modelpath)
+    model = Ebooks::Model.load(modelpath)
     statement = model.make_statement
-    log "@#{botname}: #{statement}"
-    bot = Bot.get(botname)
+    bot = Ebooks::Bot.get(botname)
     bot.configure
     bot.tweet(statement)
   end
 
-  def self.jsonify(paths)
-    usage = <<STR
-Usage: ebooks jsonify <old_corpus_path> [old_corpus_path2] [...]
+  HELP.auth = <<-STR
+    Usage: ebooks auth
 
-Takes an old-style corpus of plain tweet text and converts it to json.
-STR
+    Authenticates your Twitter app for any account. By default, will
+    use the consumer key and secret from the first defined bot. You
+    can specify another by setting the CONSUMER_KEY and CONSUMER_SECRET
+    environment variables.
+  STR
 
-    if paths.empty?
-      log usage
-      exit
+  def self.auth
+    consumer_key, consumer_secret = find_consumer
+    require 'oauth'
+
+    consumer = OAuth::Consumer.new(
+      consumer_key,
+      consumer_secret,
+      site: 'https://twitter.com/',
+      scheme: :header
+    )
+
+    request_token = consumer.get_request_token
+    auth_url = request_token.authorize_url()
+
+    pin = nil
+    loop do
+      log auth_url
+
+      log "Go to the above url and follow the prompts, then enter the PIN code here."
+      print "> "
+
+      pin = STDIN.gets.chomp
+
+      break unless pin.empty?
     end
 
-    paths.each do |path|
-      name = File.basename(path).split('.')[0]
-      new_path = name + ".json"
+    access_token = request_token.get_access_token(oauth_verifier: pin)
 
-      tweets = []
-      id = nil
-      if path.split('.')[-1] == "csv" #from twitter archive
-        csv_archive = CSV.read(path, :headers=>:first_row)
-        tweets = csv_archive.map do |tweet|
-          { text: tweet['text'], id: tweet['tweet_id'] }
-        end
-      else
-        File.read(path).split("\n").each do |l|
-          if l.start_with?('# ')
-            id = l.split('# ')[-1]
-          else
-            tweet = { text: l }
-            if id
-              tweet[:id] = id
-              id = nil
-            end
-            tweets << tweet
+    log "Account authorized successfully. Make sure to put these in your bots.rb!\n" +
+         "  access token: #{access_token.token}\n" +
+         "  access token secret: #{access_token.secret}"
+  end
+
+  HELP.console = <<-STR
+    Usage: ebooks c[onsole]
+
+    Starts an interactive ruby session with your bots loaded
+    and configured.
+  STR
+
+  def self.console
+    load_bots
+    require 'pry'; Ebooks.module_exec { pry }
+  end
+
+  HELP.start = <<-STR
+    Usage: ebooks s[tart] [botname]
+
+    Starts running bots. If botname is provided, only runs that bot.
+  STR
+
+  def self.start(botname=nil)
+    load_bots
+
+    if botname.nil?
+      bots = Ebooks::Bot.all
+    else
+      bots = Ebooks::Bot.all.select { |bot| bot.username == botname }
+      if bots.empty?
+        log "Couldn't find a defined bot for @#{botname}!"
+        exit 1
+      end
+    end
+
+    threads = []
+    bots.each do |bot|
+      threads << Thread.new { bot.prepare }
+    end
+    threads.each(&:join)
+
+    threads = []
+    bots.each do |bot|
+      threads << Thread.new do
+        loop do
+          begin
+            bot.start
+          rescue Exception => e
+            bot.log e.inspect
+            puts e.backtrace.map { |s| "\t"+s }.join("\n")
           end
+          bot.log "Sleeping before reconnect"
+          sleep 5
         end
       end
+    end
+    threads.each(&:join)
+  end
+
+  # Non-command methods
 
-      File.open(new_path, 'w') do |f|
-        log "Writing #{tweets.length} tweets to #{new_path}"
-        f.write(JSON.pretty_generate(tweets))
+  def self.find_consumer
+    if ENV['CONSUMER_KEY'] && ENV['CONSUMER_SECRET']
+      log "Using consumer details from environment variables:\n" +
+          "  consumer key: #{ENV['CONSUMER_KEY']}\n" +
+          "  consumer secret: #{ENV['CONSUMER_SECRET']}"
+      return [ENV['CONSUMER_KEY'], ENV['CONSUMER_SECRET']]
+    end
+
+    load_bots
+    consumer_key = nil
+    consumer_secret = nil
+    Ebooks::Bot.all.each do |bot|
+      if bot.consumer_key && bot.consumer_secret
+        consumer_key = bot.consumer_key
+        consumer_secret = bot.consumer_secret
+        log "Using consumer details from @#{bot.username}:\n" +
+            "  consumer key: #{bot.consumer_key}\n" +
+            "  consumer secret: #{bot.consumer_secret}\n"
+        return consumer_key, consumer_secret
       end
     end
+
+    if consumer_key.nil? || consumer_secret.nil?
+      log "Couldn't find any consumer details to auth an account with.\n" +
+          "Please either configure a bot with consumer_key and consumer_secret\n" +
+          "or provide the CONSUMER_KEY and CONSUMER_SECRET environment variables."
+      exit 1
+    end
   end
 
-  def self.command(args)
-    usage = <<STR
-Usage:
-     ebooks new <reponame>
-     ebooks consume <corpus_path> [corpus_path2] [...]
-     ebooks gen <model_path> [input]
-     ebooks score <model_path> <input>
-     ebooks archive <@user> <outpath>
-     ebooks tweet <model_path> <botname>
-     ebooks jsonify <old_corpus_path> [old_corpus_path2] [...]
-STR
+  def self.load_bots
+    load 'bots.rb'
 
+    if Ebooks::Bot.all.empty?
+      puts "Couldn't find any bots! Please make sure bots.rb instantiates at least one bot."
+    end
+  end
+
+  def self.command(args)
     if args.length == 0
-      log usage
-      exit
+      help
+      exit 1
     end
 
     case args[0]
     when "new" then new(args[1])
     when "consume" then consume(args[1..-1])
+    when "consume-all" then consume_all(args[1], args[2..-1])
     when "gen" then gen(args[1], args[2..-1].join(' '))
-    when "score" then score(args[1], args[2..-1].join(' '))
     when "archive" then archive(args[1], args[2])
     when "tweet" then tweet(args[1], args[2])
     when "jsonify" then jsonify(args[1..-1])
+    when "auth" then auth
+    when "console" then console
+    when "c" then console
+    when "start" then start(args[1])
+    when "s" then start(args[1])
+    when "help" then help(args[1])
     else
-      log usage
+      log "No such command '#{args[0]}'"
+      help
       exit 1
     end
   end
 end
 
-Ebooks.command(ARGV)
+Ebooks::CLI.command(ARGV)