elephrame 0.4.6 → 0.5

data/examples/tracery_simple.rb

@@ -1,5 +1,7 @@
 require 'elephrame'
 
+# we define our bot by telling elephrame how often it should post,
+#  and where it should load our tracery rules from
 bot = Elephrame::Bots::TraceryBot.new('10s', 'tracery_files')
 
 #  this code happens automatically in the framework

data/lib/elephrame/mix/bots.rb

@@ -1,6 +1,7 @@
 require_relative '../rest/rest'
 require_relative '../streaming/streaming'
 require_relative './tracery'
+require_relative './generative'
 require_relative '../bot'
 
 module Elephrame
@@ -62,17 +63,17 @@ module Elephrame
       ##
       # create a new TraceryBot
       # @param interval [String] a string representing the interval to post
-      # @param tracery_dir [String] a string with the path to the directory
-      #    containing all of the tracery grammer rules.
+      # @param dirs [Array<String>] an array of strings with paths to  directories
+      #    containing tracery grammer rules
       # @return [Elephrame::Bots::TraceryBot]
       
-      def initialize interval, tracery_dir
+      def initialize interval, *dirs
         super()
 
         # set up our bot stuff
         setup_scheduler interval
         setup_streaming
-        setup_tracery tracery_dir
+        setup_tracery dirs
       end
 
       ##
@@ -82,13 +83,322 @@ module Elephrame
       
       def run
         run_scheduled &Proc.new
+
+        # if we have any logic for on_reply, we run that
+        #  otherwise we go past it and wait for our scheduler to finish
+        run_reply unless @on_reply.nil?
+        @scheduler.join
+      end
+    end
+
+    ##
+    # A basic Ebooks bot template
+    
+    class EbooksBot < GenerativeBot
+      attr :update_interval,
+           :old_id,
+           :scrape_filter
+      
+      PrivacyLevels = ['public', 'unlisted', 'private', 'direct']
+      APILimit = 280
+      RetryTime = '6m'
+      
+      ##
+      # Creates a new Ebooks bot
+      #
+      # @param interval [String] how often should the bot post on it's own
+      # @param opts [Hash] options for the bot
+      # @option opt cw [String]
+      # @option opt update_interval [String] how often to scrape new posts
+      #        from the accounts the bot follows
+      # @option opt retry_limit [Integer] the amount of times to retry
+      #        generating a post
+      # @option opt model_filename [String] path to a file where we 
+      #        will save our backing ebooks model data
+      # @option opt filter_filename [String] path to a file where we 
+      #        will save our internal filtered words data      
+      # @option opt visibility [String] the posting level the bot will default to
+      # @option opt scrape_privacy [String] the highest privacy the bot should
+      #        scrape for content
+      
+      def initialize(interval, opts = {})
+        super
+
+        # add our manual update command
+        add_privileged_command 'update' do
+          fetch_new_posts
+        end
+
+        # set some defaults for our internal vars
+        level = PrivacyLevels.index(opts[:scrape_privacy]) || 0
+        @scrape_filter = /(#{PrivacyLevels[0..level].join('|')})/
+        @update_interval = opts[:update_interval] || '2d'
+
+        # if we don't have what a newest post id then we fetch them
+        #  for each account
+        if @model_hash[:last_id].empty?
+          @old_id = {}
+          
+          @following.each do |account|
+            # get the newest post from this account and save the id
+            newest_id = @client.statuses(account,
+                                         exclude_reblogs: true,
+                                         limit: 1).first.id
+            @model_hash[:last_id][account] = newest_id
+            @old_id[account] = newest_id
+          end
+        end
+
+        # if our model's token are empty that means we have an empty model
+        fetch_old_posts if @model_hash[:model].tokens.empty?
+      end
+
+      ##
+      # Method to go and fetch all posts
+      #  should be ran first
+      
+      def fetch_old_posts
+        begin
+          # init some vars to keep track of where we are
+          api_calls = 1
+          errored = false
+          new_posts = { statuses: [],
+                        mentions: [] }
+
+          # for each account we're following
+          @following.each do |account|
+            # okay so
+            #  we keep track of how many get requests we're doing and before
+            #  the limit (300) we schedule for 5min and go on, saving what we got
+            posts = @client.statuses(account,
+                                     exclude_reblogs: true,
+                                     limit: 40,
+                                     max_id: @old_id[account])
+            
+            # while we still have posts and haven't gotten near the api limit
+            while not posts.size.zero? and api_calls < APILimit
+              posts.each do |post|
+                
+                # add the new post to our hash
+                if post.visibility =~ @scrape_filter
+                  new_posts = add_post_to_hash post, new_posts
+                end
+                
+                # set our cached id to the latest post id
+                @old_id[account] = post.id
+              end
+              
+              # fetch more posts
+              posts = @client.statuses(account,
+                                       exclude_reblogs: true,
+                                       limit: 40,
+                                       max_id: @old_id[account])
+              api_calls += 1
+            end
+            
+            break if api_calls >= APILimit
+          end
+          
+        rescue 
+          errored = true
+          
+        ensure
+          # consume our posts, and then save our model
+          @model_hash[:model].consume! new_posts
+          save_file(@model_filename,
+                    @model_hash.collect {|key, value| value.to_hash }.to_yaml)
+
+          # if we have more than our limit of api calls
+          #  or we errored out that means we need to check again
+          if api_calls >= APILimit or errored 
+            @scheduler.in RetryTime do
+              fetch_old_posts
+            end
+          end
+        end
+      end
+      
+      ##
+      # Fetch posts from the accounts the bot follows
+      
+      def fetch_new_posts
+        begin
+          # set up some vars for tracking our progress
+          added_posts = { statuses: [],
+                          mentions: [] }
+          api_calls = 1
+          errored = false
+
+          # for each account we're following
+          @following.each do |account|
+            # get 40 posts at a time, where we left off
+            posts = @client.statuses(account,
+                                     exclude_reblogs: true,
+                                     limit: 40,
+                                     since_id: @model_hash[:last_id][account])
+
+            # while we have posts to process and we haven't
+            #  gotten near the api limit
+            while not posts.size.zero? and api_calls < APILimit
+              posts.reverse_each do |post|
+                # save our post id for next loop
+                @model_hash[:last_id][account] = post.id
+
+                # if the post matches our set visibility we add it to our hash
+                if post.visibility =~ @scrape_filter
+                  added_posts = add_post_to_hash post, added_posts
+                end
+              end
+
+              # fetch more posts
+              posts = @client.statuses(account,
+                                       exclude_reblogs: true,
+                                       limit: 40,
+                                       since_id: @model_hash[:last_id][account])
+              api_calls += 1
+            end
+
+            # in case we hit our api limit between calls
+            break if api_calls >= APILimit
+          end
+          
+        rescue 
+          # if we've hit here then we've errored out
+          errored = true
+          
+        ensure
+          # consume our new posts, and add them to our original hash
+          @model_hash[:model].consume! added_posts
+          
+          if api_calls >= APILimit or errored
+            @scheduler.in RetryTime do
+              fetch_new_posts
+            end
+          end
+          
+          # then we save 
+          save_file(@model_filename,
+                    @model_hash.collect {|key, value| value.to_hash }.to_yaml)
+        end
+      end
+
+      ##
+      # Run the Ebooks bot
+      
+      def run
+        # set up our scheduler to scrape posts
+        @scheduler.repeat @update_interval do
+          fetch_new_posts
+        end
+
+        # call generativebot's run method
+        super
+      end
+
+      private
+
+      ##
+      # adds a post into the +post_hash+ hash
+      #  makes sure it gets put under the appropriate key
+      #
+      # @param post [Mastodon::Status]
+      
+      def add_post_to_hash post, hash
+        # make sure we strip out the html crap
+        post.class
+          .module_eval { alias_method :content, :strip } if @strip_html
         
-        unless @on_reply.nil?
-          run_reply
+        # decide which array the post should go into, based
+        #  on if it's a reply or not
+        # also make sure to strip out any account names
+        if post.in_reply_to_id.nil? or post.mentions.size.zero?
+          hash[:statuses] << post.content
         else
-          @scheduler.join
+          hash[:mentions] << post.content.gsub(/@.+?(@.+?)?\s/, '')
+        end
+
+        hash
+      end
+    end
+
+    ##
+    # A more general purpose markov bot. Reads in data from a supplied source
+    
+    class MarkovBot < GenerativeBot
+
+      ##
+      # Creates a new Ebooks bot
+      #
+      # @param interval [String] how often should the bot post on it's own
+      # @param sources [Array] all of the sources for the bot. either 
+      #        folders or files
+      # @param opts [Hash] options for the bot
+      # @option opt cw [String]
+      # @option opt retry_limit [Integer] the amount of times to retry
+      #        generating a post
+      # @option opt model_filename [String] path to a file where we 
+      #        will save our backing ebooks model data
+      # @option opt filter_filename [String] path to a file where we 
+      #        will save our internal filtered words data
+      # @option opt visibility [String] the posting level the bot will default to
+      
+      def initialize(interval, *sources, **options)
+        super(interval, options)
+
+        raise 'no sources provided!' if sources.empty?
+        
+        # initialize the model to contain the specified source text
+        if @model_hash[:model].tokens.empty?
+          sources.each do |source|
+            if Dir.exists? source
+              Dir.open source do |file|
+                next if file =~ /^\.\.?$/
+                read_and_consume "#{source}/#{file}"
+              end
+            elsif File.exists? source
+              read_and_consume source
+            else
+              raise "source #{source} could not be loaded"
+            end
+          end
+
+          save_file(@model_filename,
+                    @model_hash[:model].to_hash.to_yaml)
         end
       end
+
+      private
+
+      ##
+      # reads a file in and adds it into the model
+      #
+      # @param file [String] path to a file
+      
+      def read_and_consume file
+        @model_hash[:model].consume! File.read(file)
+      end
+
+      
+=begin
+      ##
+      # scrapes text from a provided url
+      #
+      # @param url [String] a url
+      # @returns [Boolean]
+      
+      def download_and_consume url
+        uri = URI.parse url
+        errored = false
+        
+        begin
+          
+        rescue
+          errored = true
+        end
+
+        errored
+      end
+=end
     end
   end
 end

data/lib/elephrame/mix/generative.rb

@@ -0,0 +1,244 @@
+module Elephrame
+  module Bots
+    class GenerativeBot < BaseBot
+      include Elephrame::Streaming
+      include Elephrame::Reply
+      include Elephrame::Scheduler
+      include Elephrame::Command
+      
+      attr_accessor :cw
+      attr :filter,
+           :filter_words,
+           :filter_by,
+           :following,
+           :model,
+           :char_limit,
+           :retry_limit,
+           :visibility,
+           :model_hash,
+           :model_filename,
+           :filter_filename
+      
+      backup_method :post, :actually_post
+      SavedFileName = 'model.yml'
+      SavedFilterFileName = 'filter.yml'
+      
+      def initialize(interval, options = {})
+        require 'moo_ebooks'
+        require 'yaml'
+        
+        # initialize our botness
+        super()
+        
+        # setup our various classes
+        setup_streaming
+        setup_scheduler interval
+        setup_command
+        
+        # set some defaults and initialize some vars
+        @model_hash = { model: Ebooks::Model.new,
+                        last_id:  {} }
+        @filter = /^$/
+        @filter_words = []
+        @following = []
+        @char_limit = @client.instance.max_toot_chars || 500
+        @retry_limit = options[:retry_limit] || 10
+        @cw = options[:cw] || 'markov post'
+        @visibility = options[:visibility] || 'unlisted'
+        @model_filename = options[:model_filename] || SavedFileName
+        @filter_filename = options[:filter_filename] || SavedFilterFileName
+        
+        # load our model if it exists
+        if File.exists? @model_filename
+          values = load_file(@model_filename)
+          @model_hash[:model] = Ebooks::Model.from_hash(values.first)
+          @model_hash[:last_id] = values.last
+        end
+        
+        @filter_words = load_file(@filter_filename) if File.exists? @filter_filename
+        
+        # add our default commands
+        #
+        # !delete will delete the status it's in reply to
+        add_privileged_command 'delete' do |bot, content, status|
+          @client.destroy_status(status.in_reply_to_id)
+        end
+        
+        # !filter will add every word from the post into the word filter
+        add_privileged_command 'filter' do |bot, content, status|
+          content.split.each do |word|
+            add_filter_word word
+          end
+          save_file @filter_filename, @filter_words.to_yaml
+          bot.reply("'#{content}' added to internal filter")
+        end
+
+        # add a help command that explains the other commands
+        add_privileged_command 'help' do |bot|
+          bot.reply(default_help)
+        end
+        
+        # set up a default for replying
+        on_reply do |bot, status|
+          # retry our status creation until we get something that
+          #  passes our filters
+          @retry_limit.times do
+            text = @model_hash[:model].reply(status
+                                               .content
+                                               .gsub(/@.+?(@.+?)?\s/, ''),
+                                             @char_limit)
+            break unless bot.reply_with_mentions(text,
+                                                 spoiler: @cw).nil?
+          end
+        end
+        
+        # get our own account id and save the ids of the accounts
+        #  we're following
+        acct_id = @client.verify_credentials.id
+        @client.following(acct_id).each do |account|
+          @following << account.id
+        end
+      end
+      
+      ##
+      # Runs the bot
+      
+      def run
+        # see scheduler.rb
+        run_scheduled do |bot|
+          @retry_limit.times do
+            text = @model_hash[:model].update(@char_limit)
+            break unless bot.post(text,
+                                  spoiler: @cw,
+                                  visibility: @visibility).nil?
+          end
+        end
+        
+        # we do this because run_commands accepts a block that
+        #  will run when it doesn't find a command in a mention
+        #  this should work. :shrug:
+        run_commands do |bot, status|
+          @on_reply.call(bot, status)
+        end
+      end
+
+      ##
+      # generates a default help message for the default commands
+      # if you add custom commands add a `custom_command_help` method
+      # that returns a string. it will be added to the end of this
+      #
+      # @returns [String] default help text
+
+      def default_help
+        txt = []
+        txt << "#{@prefix}delete -- deletes the status that the command post is replying to"
+        txt << "#{@prefix}filter -- adds all words from the command post into the internal filter"
+        txt << "#{@prefix}help -- replies with this help text"
+        txt << custom_command_help if respond_to? :custom_command_help
+        txt.join "\n"
+      end
+
+      ##
+      # adds a command that can only be executed by someone
+      #  that the bot follows
+      #
+      # @param cmd [String] a command to add
+      # @param block [Proc] the code to execute when +cmd+ is recieved
+      
+      def add_privileged_command cmd, &block
+        add_command cmd do |bot, content, status|
+          if @following.include? status.account.id
+            block.call(bot, content, status)
+          end
+        end
+      end
+      
+      ##
+      # loads a yaml file containing our model data
+      #
+      # @param filename [String] file to read in from
+      
+      def load_file filename
+        YAML.load_file(filename)
+      end
+      
+      ##
+      # Saves a yaml file containing our model data
+      #
+      # @param filename [String] file to write out to
+      
+      def save_file filename, data
+        File.write(filename, data)
+      end
+      
+      ##
+      # Sets the filter regex
+      #  if arg is a string array, 'or's the strings together
+      #  if it's a regexp it just sets it to the value
+      #
+      # @param arg [Array<String>,String,Regexp]
+      
+      def filter= arg
+        arg = arg.join('|') if arg.kind_of? Array
+        arg = /#{arg}/ unless arg.kind_of? Regexp
+        @filter = arg
+      end
+      
+      ##
+      # Returns a string representing all of the current
+      #  words being checked in the filter
+      #
+      # @returns [String] comma separated list of all filter words
+      
+      def filter_words
+        @filter_words.join(', ')
+      end
+      
+      ##
+      # Adds a word into the filter list
+      #
+      # @param word [String]
+      
+      def add_filter_word(word)
+        @filter_words << word
+        filter = @filter_words
+      end
+      
+      ##
+      # Accepts a block to check the post against before posting
+      #
+      # @param block [Proc]
+      
+      def filter_by &block
+        @filter_by = block
+      end
+      
+      ##
+      # Checks the proposed post against the filters
+      #  only posts if the text passes the filters
+      #
+      # @param text [String] the tracery text to expand before posting
+      # @param options [Hash] a hash of arguments to pass to post
+      # @option options rules [String] the grammar rules to load
+      # @option options visibility [String] visibility level
+      # @option options spoiler [String] text to use as content warning
+      # @option options reply_id [String] id of post to reply to
+      # @option options hide_media [Bool] should we hide media?
+      # @option options media [Array<String>] array of file paths
+      
+      def filter_and_post(text, *options)
+        opts = Hash[*options]
+        
+        # default passed to false and then see if
+        #  the supplied text gets through our filters
+        passed = false
+        passed = !(text =~ @filter)
+        passed = @filter_by.call(text) unless @filter_by.nil?
+        
+        actually_post(text, **opts) if passed
+      end
+      
+      alias_method :post, :filter_and_post
+    end
+  end
+end

data/lib/elephrame/mix/tracery.rb

@@ -12,29 +12,32 @@ module Elephrame
     # loads all of our tracery files into our +files+ hash
     # if a file is named 'default' then we load that into +grammar+
     #
-    # @param dir [String] path to the directory containing the tracery rules
+    # @param dirs [String] path to the directory containing the tracery rules
     
-    def setup_tracery dir_path
+    def setup_tracery *dirs
       raise "Provided path not a directory" unless Dir.exist?(dir_path)
 
       @grammar = {}
-      Dir.open(dir_path) do |dir|
-        dir.each do |file|
-          # skip our current and parent dir
-          next if file =~ /^\.\.?$/
 
-          # read the rule file into the files hash
-          @grammar[file.split('.').first] =
-            createGrammar(JSON.parse(File.read("#{dir_path}/#{file}")))
+      dirs.each do |directory|
+        Dir.open(directory) do |dir|
+          dir.each do |file|
+            # skip our current and parent dir
+            next if file =~ /^\.\.?$/
+            
+            # read the rule file into the files hash
+            @grammar[file.split('.').first] =
+              createGrammar(JSON.parse(File.read("#{dir_path}/#{file}")))
+          end
         end
       end
-
+        
       # go ahead and makes a default mention-handler
       #  if we have a reply rule file
       unless @grammar['reply'].nil?
-        on_reply { |bot|
+        on_reply do |bot|
           bot.reply_with_mentions('#default#', rules: 'reply')
-        }
+        end
       end
     end
 

data/lib/elephrame/rest/rest.rb

@@ -23,7 +23,7 @@ module Elephrame
         @schedule = j
         yield(self)
       end
-      @scheduler.join unless not @streamer.nil?
+      @scheduler.join if @streamer.nil?
     end
 
     alias_method :run, :run_scheduled

data/lib/elephrame/streaming/command.rb

@@ -35,7 +35,7 @@ module Elephrame
     # @param usage [String] 
     
     def set_help usage
-      add_command 'help' do |bot, content, status|
+      add_command 'help' do |bot|
         bot.reply("#{usage}")
       end
     end

data/lib/elephrame/streaming/reply.rb

@@ -43,7 +43,7 @@ module Elephrame
       #  unless that account is our own, or the tagged account
       #  has #NoBot
       mentions = @mention_data[:mentions].collect do |m|
-        "@#{m.acct}" unless m.acct == @username or no_bot? m.id
+        "@#{m.acct}" unless m.acct == @username or @client.account(m.id).no_bot?
       end.join ' '
       
       reply("#{mentions.strip} #{text}", *options)

data/lib/elephrame/streaming/streaming.rb

@@ -11,8 +11,9 @@ module Elephrame
     # Creates the stream client
     
     def setup_streaming
-      stream_uri = @client.instance()
-                     .attributes['urls']['streaming_api'].gsub(/^wss?/, 'https')
+      stream_uri = @client.instance
+                     .attributes['urls']['streaming_api']
+                     .gsub(/^wss?/, 'https')
       @streamer = Mastodon::Streaming::Client.new(base_url: stream_uri,
                                                   bearer_token: ENV['TOKEN'])
     end