replacer_bot 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c642a22ef658553bbc3f6fc8c322bda2a86745f
-  data.tar.gz: e3fd74ac7ae4cba27a31d4cb269a971631ccbf81
+  metadata.gz: d9cc0084fcde5c232d3ffe37093ef1c95e1556e5
+  data.tar.gz: 0363fabbc0feb1e3a9b8a2fb08a06cf207120628
 SHA512:
-  metadata.gz: 833d985c3c110abb905e5fb5020c1c36462970508a36babb94e213d7613cb4caafdd11d807ab3504b4723d0114a266c78d15e506cddef7a4cf65c5ba51c10abf
-  data.tar.gz: 2c0444ec1bce07cc3c844e1baa9643dbfc2092bfe246610a6de38eccb746408f18603c2b81ad832b9df0c17920c8873813db543e0714f59c3f4497a706912779
+  metadata.gz: 506d9982aa39ac4198556ed6b42bed26d6305f117a64bccb8df617b45eb74f311e3c259a46c237dc9bcbb5dd29ab98334fa5174d24b66c1e5d1545dc235e45a5
+  data.tar.gz: 0e9bacffd8e373d1e05c7082d90fbbdb7f0c662e8b49eeef00ff4a96f33a88316955036c6e366aaf70ea4b1ac7af830d04f955133723ceefcd2794a64d9a7065

data/.gitignore

@@ -15,3 +15,4 @@ mkmf.log
 
 .env
 last.tweet
+derp.rb

data/README.md

@@ -27,7 +27,7 @@ or
 
 ## Configuration
 
-The default config is [here](https://github.com/pikesley/replacer_bot/blob/master/config/defaults.yml), you'll want to create your own config at `~/.replacer_bot/config.yml` to override come of these, something like:
+The default config is [here](https://github.com/pikesley/replacer_bot/blob/master/config/defaults.yml), you'll want to create your own config at `~/.replacer_bot/config.yml` to override some of these, something like:
 
     search_term: David Cameron
     replacements:
@@ -35,9 +35,10 @@ The default config is [here](https://github.com/pikesley/replacer_bot/blob/maste
       - cameron: Satan
     save_file: /Users/sam/.replacer_bot/last.tweet
 
-Note:
+Notes:
 
 * The search-and-replace terms will be applied in the order listed, which you may or may not care about
+* The search part of the search-and-replace is case-insensitive
 
 You'll also need some Twitter credentials, store them in `~/.replacer_botrc` like this:
 
@@ -50,11 +51,19 @@ You'll also need some Twitter credentials, store them in `~/.replacer_botrc` lik
 
 ## Running it
 
-You should now be able to do run it like so:
+You should now be able to run it like so:
 
     ➔ replacer tweet
     Tweeting: Satan's Little Helper sets out academy 'vision' for every school http://t.co/S6yFWRf7pD
     Sleeping 60 seconds
     Tweeting: Swarm warning: Satan's Little Helper accuses migrants of 'breaking in' to UK http://t.co/1sB5J8Alwi
 
-There's also a `dry_run` command, which shows you what it would have tweeted
+Notes:
+
+* Direct replies and manual retweets are excluded
+
+There's also
+
+    ➔ replacer dry_run
+
+which does the search and shows what it would have tweeted, without actually tweeting anything

data/Rakefile

@@ -7,4 +7,4 @@ Coveralls::RakeTask.new
 RSpec::Core::RakeTask.new
 Cucumber::Rake::Task.new
 
-task :default => [:spec, 'coveralls:push']
+task :default => [:spec, :cucumber, 'coveralls:push']

data/config/defaults.yml

@@ -2,6 +2,7 @@ search_term: open data
 search_count: 100
 ignore_spaces: true
 save_file: last.tweet
+seen_tweets: seen.tweets
 replacements:
   - open data: Taylor Swift
   - opendata: TaylorSwift

data/features/support/env.rb

@@ -1,2 +1,5 @@
 require 'aruba/cucumber'
 require 'cucumber/rspec/doubles'
+
+require 'coveralls'
+Coveralls.wear!

data/lib/replacer_bot.rb

@@ -9,6 +9,7 @@ require 'replacer_bot/version'
 require 'replacer_bot/replacer'
 require 'replacer_bot/config'
 require 'replacer_bot/helpers'
+require 'replacer_bot/seen_tweets'
 require 'replacer_bot/twitter_client'
 
 Dotenv.load

data/lib/replacer_bot/helpers.rb

@@ -1,8 +1,12 @@
 module ReplacerBot
-  def self.encode term
+  def self.encode term:
     URI.encode "\"#{term}\""
   end
 
+  def self.is_hashtag word
+    word[0] == '#'
+  end
+
   def self.last_tweet
     begin
       Marshal.load File.read Config.instance.config.save_file
@@ -13,23 +17,23 @@ module ReplacerBot
     end
   end
 
-  def self.validate string, term = Config.instance.config.search_term, ignore_spaces = true
+  def self.validate string:, term: Config.instance.config.search_term, ignore_spaces: true
     return false if string[0...2] == 'RT'
     return false if string[0] == '@'
 
     term = term.gsub ' ', ' ?' if ignore_spaces
-    return true if string.index /#{term}/i
+    return true if string.index(/#{term}/i) && SeenTweets.validate(string)
 
     false
   end
 
-  def self.filter list, ignore_spaces = true
-    list.select { |i| self.validate i.text, Config.instance.config.search_term, ignore_spaces }.
+  def self.filter list:, ignore_spaces: true
+    list.select { |i| self.validate string: i.text, term: Config.instance.config.search_term, ignore_spaces: ignore_spaces }.
       select { |i| i.id > self.last_tweet}
   end
 
   def self.dehash word
-    if word[0] == '#'
+    if is_hashtag word
       return word[1..-1]
     end
 
@@ -76,7 +80,7 @@ module ReplacerBot
     ]
   end
 
-  def self.replace string, subs = Config.instance.config.replacements
+  def self.replace string:, subs: Config.instance.config.replacements
     # Something about a frozen string
     our_string = string.dup
     subs.each do |substitute|

data/lib/replacer_bot/replacer.rb

@@ -10,12 +10,12 @@ module ReplacerBot
 
     def search #count = 20
       @results ||= begin
-        results = ReplacerBot.filter @client.search(ReplacerBot.encode(@search_term), result_type: 'recent').take(@config.search_count), @config.ignore_spaces
+        results = ReplacerBot.filter list: @client.search(ReplacerBot.encode(term: @search_term), result_type: 'recent').take(@config.search_count), ignore_spaces: @config.ignore_spaces
       end
     end
 
     def tweets
-      search.map { |r| ReplacerBot.truncate ReplacerBot.replace r.text }
+      search.map { |r| ReplacerBot.truncate ReplacerBot.replace string: r.text }
     end
 
     def tweet dry_run: false, chatty: false

data/lib/replacer_bot/seen_tweets.rb

@@ -0,0 +1,60 @@
+module ReplacerBot
+  class SeenTweets
+    def self.validate tweet
+      archive = retrieve
+      t = sanitise tweet
+      valid = not(archive.include? t)
+      archive.add t
+      save archive
+
+      valid
+    end
+
+    def self.retrieve
+      begin
+        Marshal.load File.open Config.instance.config.seen_tweets
+      rescue Errno::ENOENT
+        Set.new
+      end
+    end
+
+    def self.clean_urls string
+      string.gsub /https?:\/\/[^ ]*/, '__URL__'
+    end
+
+    def self.hashtag_nuker string:, other_end: false
+      words = string.split ' '
+      words.reverse! if other_end
+
+      no_hashtag_yet = false
+
+      a = []
+      words.each do |token|
+        unless ReplacerBot.is_hashtag token
+          no_hashtag_yet = true
+        end
+
+        if no_hashtag_yet
+          a.push token
+        end
+      end
+
+      a.reverse! if other_end
+      a.join ' '
+    end
+
+    def self.nuke_hashtags string
+      hashtag_nuker string: (hashtag_nuker string: string, other_end: true)
+    end
+
+    def self.sanitise tweet
+      nuke_hashtags clean_urls tweet
+    end
+
+    def self.save set
+      File.open Config.instance.config.seen_tweets, 'w' do |file|
+        Marshal.dump set, file
+      end
+    end
+  end
+end

data/lib/replacer_bot/version.rb

@@ -1,3 +1,3 @@
 module ReplacerBot
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/lib/replacer_bot/helpers_spec.rb

@@ -1,8 +1,13 @@
 module ReplacerBot
   describe 'Helpers' do
+    after :each do
+      FileUtils.rm_f Config.instance.config.save_file
+      FileUtils.rm_f Config.instance.config.seen_tweets
+    end
+
     context 'URLs' do
       it 'URL-encodes a search term' do
-        expect(ReplacerBot.encode 'open data').to eq '%22open%20data%22'
+        expect(ReplacerBot.encode term: 'open data').to eq '%22open%20data%22'
       end
     end
 
@@ -22,23 +27,23 @@ module ReplacerBot
 
     context 'filtering' do
       it 'validates liberally' do
-        expect(ReplacerBot.validate 'opendata hulk', 'open data').to eq true
+        expect(ReplacerBot.validate string: 'opendata hulk', term: 'open data').to eq true
       end
 
       it 'validates more strictly' do
-        expect(ReplacerBot.validate 'open data ftw', 'open data', ignore_spaces = false).to eq true
-        expect(ReplacerBot.validate 'i love opendata', 'open data', ignore_spaces = false).to eq false
+        expect(ReplacerBot.validate string: 'open data ftw', term: 'open data', ignore_spaces: false).to eq true
+        expect(ReplacerBot.validate string: 'i love opendata', term: 'open data', ignore_spaces: false).to eq false
       end
 
       it 'validates away rubbish' do
-        expect(ReplacerBot.validate 'incredible hulk', 'open data').to eq false
+        expect(ReplacerBot.validate string: 'incredible hulk', term: 'open data').to eq false
       end
 
       it 'filters retweets' do
-        expect(ReplacerBot.validate 'RT @xyz This is about Open Data').to eq false
+        expect(ReplacerBot.validate string: 'RT @xyz This is about Open Data').to eq false
       end
       it 'filters direct replies' do
-        expect(ReplacerBot.validate '@abc This is a reply about Open Data').to eq false
+        expect(ReplacerBot.validate string: '@abc This is a reply about Open Data').to eq false
       end
     end
 
@@ -60,8 +65,8 @@ module ReplacerBot
       end
 
       it 'replaces text' do
-        expect(ReplacerBot.replace 'Something about Open Data goes here').to eq 'Something about Taylor Swift goes here'
-        expect(ReplacerBot.replace 'Something about #opendata http://foo.bar/').to eq 'Something about #TaylorSwift http://foo.bar/'
+        expect(ReplacerBot.replace string: 'Something about Open Data goes here').to eq 'Something about Taylor Swift goes here'
+        expect(ReplacerBot.replace string: 'Something about #opendata http://foo.bar/').to eq 'Something about #TaylorSwift http://foo.bar/'
       end
 
       it 'does a/an correctly' do
@@ -80,8 +85,13 @@ module ReplacerBot
       end
 
       it 'uses the correct article in replacements' do
-        expect(ReplacerBot.replace 'This is an Open Data tweet').to eq 'This is a Taylor Swift tweet'
-        expect(ReplacerBot.replace 'This is an Open Data tweet about an #opendata story').to eq 'This is a Taylor Swift tweet about a #TaylorSwift story'
+        expect(ReplacerBot.replace string: 'This is an Open Data tweet').to eq 'This is a Taylor Swift tweet'
+        expect(ReplacerBot.replace string: 'This is an Open Data tweet about an #opendata story').to eq 'This is a Taylor Swift tweet about a #TaylorSwift story'
+      end
+
+      it 'recognises a hashtag' do
+        expect(ReplacerBot.is_hashtag '#hashtag').to eq true
+        expect(ReplacerBot.is_hashtag 'not_hashtag').to eq false
       end
     end
   end

data/spec/lib/replacer_bot/replacer_spec.rb

@@ -1,7 +1,8 @@
 module ReplacerBot
   describe Replacer do
     after :each do
-      FileUtils.rm_f 'last.tweet'
+      FileUtils.rm_f Config.instance.config.save_file
+      FileUtils.rm_f Config.instance.config.seen_tweets
     end
 
     context 'search' do
@@ -54,18 +55,27 @@ module ReplacerBot
       end
     end
 
+    context 'filtering on similar tweets' do
+      let(:replacer) { described_class.new }
+
+      it 'filters similar tweets', :vcr do
+        SeenTweets.validate 'How open data can help save lives http://t.co/90U7bVq5UF'
+        expect(replacer.tweets.count).to eq 19
+      end
+    end
+
     context 'tweet' do
       let(:replacer) { described_class.new }
 
       it 'prepares sensible tweets', :vcr do
         expect(replacer.tweets).to be_a Array
         expect(replacer.tweets.first).to eq 'Taylor Swift Hackathon 6-7 октября'
-        expect(replacer.tweets[27]).to eq 'CompTIA_SLED: RT CADeptTech: Building a User-Centric #California #Government through Demand-Driven #TaylorSwift http://t.co/mh91wbk4xK ---…'
+        expect(replacer.tweets[10]).to eq 'Lovely: "Does Taylor Swift Build Trust?" by @denicewross https://t.co/zcuOX6O8pA'
         expect(replacer.tweets.all? { |t| t.length <= 140} ).to eq true
       end
 
       it 'actually sends tweets', :vcr do
-        expect(replacer.client).to(receive(:update)).exactly(21).times
+        expect(replacer.client).to(receive(:update)).exactly(18).times
         interval = replacer.config.interval
         replacer.config.interval = 0
         replacer.tweet

data/spec/lib/replacer_bot/seen_tweets_spec.rb

@@ -0,0 +1,96 @@
+module ReplacerBot
+  describe SeenTweets do
+    after :each do
+      FileUtils.rm_f Config.instance.config.seen_tweets
+    end
+
+    context 'sanitise' do
+      it 'blanks out URLs' do
+        expect(described_class.clean_urls 'Some text with http://foo.bar/ in it').to eq 'Some text with __URL__ in it'
+        expect(described_class.clean_urls 'Other text with https://foo.bar/?123 and http://example.com/derp#fragment in it').to eq 'Other text with __URL__ and __URL__ in it'
+        expect(described_class.clean_urls 'Some text without any URLs in it').to eq 'Some text without any URLs in it'
+      end
+
+      it 'removes hashtags from the end of text' do
+        expect(described_class.nuke_hashtags 'Text finishing with a #hashtag').to eq 'Text finishing with a'
+        expect(described_class.nuke_hashtags 'This embedded #hashtag should survive but not this one #spurious').
+        to eq 'This embedded #hashtag should survive but not this one'
+      end
+
+      it 'removes hashtags from the beginning of text' do
+        expect(described_class.nuke_hashtags '#Beginning hashtag should go away').to eq 'hashtag should go away'
+      end
+
+      it 'strips hashtags at either end but leaves embedded ones' do
+        expect(described_class.nuke_hashtags '#This #will go away #but then #also #these').
+          to eq 'go away #but then'
+      end
+
+      it 'returns nothing if all it gets is hashtags' do
+        expect(described_class.nuke_hashtags '#nothing #but #hashtags').to eq ''
+      end
+
+      it 'sanitises tweets' do
+        expect(described_class.sanitise '#Hashtag at the start with http://derp.com/thing #this and also #these').
+          to eq 'at the start with __URL__ #this and also'
+      end
+    end
+
+    it 'validates the first tweet' do
+      expect(described_class.validate 'This is a tweet').to eq true
+    end
+
+    it 'invalidates on seeing the same tweet again' do
+      described_class.validate 'This is a tweet'
+      expect(described_class.validate 'This is a tweet').to eq false
+    end
+
+    it 'invalidates similar tweets with different URLs' do
+      described_class.validate 'This is a tweet with https://foo.bar/abcd'
+      expect(described_class.validate 'This is a tweet with https://foo.bar/xyz').to be false
+    end
+
+    it 'invalidates similar tweets laden with hashtags' do
+      described_class.validate 'This is a tweet'
+      expect(described_class.validate 'This is a tweet #loaded #with #hashtags').to be false
+    end
+
+    it 'validates and invalidates correctly' do
+      corpus = [
+        'This is a tweet with #hashtag https://derp.com/abc #trailing #tags',
+        'This is a different tweet',
+        '#Needless #hashtags tacked on to #this tweet'
+      ]
+      corpus.each do |tweet|
+        described_class.validate tweet
+      end
+
+      test_cases = {
+        'This one should be fine' => true,
+        'This is a different tweet #with #hashtags' => false,
+        '#Different #tags tacked on to #this tweet #here' => false,
+        'This is a tweet with #hashtag http://what.even/' => false,
+        'This is a tweet with #hashtag http://what.even/xyz #derp' => false
+      }
+      test_cases.each_pair do |tweet, expectation|
+        expect(described_class.validate tweet).to eq expectation
+      end
+
+      expect(described_class.retrieve.to_a.sort).to eq [
+        "This is a different tweet",
+        "This is a tweet with #hashtag __URL__",
+        "This one should be fine",
+        "tacked on to #this tweet"
+      ]
+    end
+
+    it 'saves a set' do
+      set = Set.new [1, 2, 3]
+      described_class.save set
+
+      expect(Marshal.load File.open Config.instance.config.seen_tweets).to be_a Set
+      expect(Marshal.load File.open Config.instance.config.seen_tweets).to include 1
+      expect(Marshal.load File.open Config.instance.config.seen_tweets).to include 3
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,3 +1,6 @@
+require 'coveralls'
+Coveralls.wear!
+
 require 'replacer_bot'
 require_relative 'support/vcr_setup'