replacer_bot 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9cc0084fcde5c232d3ffe37093ef1c95e1556e5
-  data.tar.gz: 0363fabbc0feb1e3a9b8a2fb08a06cf207120628
+  metadata.gz: b1e8d3be70c7dfae8451a8a7e6655bb74bb545a9
+  data.tar.gz: 062f05a23f866b7eb168f95160b7dc5d07a4632d
 SHA512:
-  metadata.gz: 506d9982aa39ac4198556ed6b42bed26d6305f117a64bccb8df617b45eb74f311e3c259a46c237dc9bcbb5dd29ab98334fa5174d24b66c1e5d1545dc235e45a5
-  data.tar.gz: 0e9bacffd8e373d1e05c7082d90fbbdb7f0c662e8b49eeef00ff4a96f33a88316955036c6e366aaf70ea4b1ac7af830d04f955133723ceefcd2794a64d9a7065
+  metadata.gz: 830a1b44dfc123d1ea276ef70deecd5f80276dd9900f43bc6d6c3d7024f87387b0b2acedd26197b028ba8d83509438bb354366bd555219fcf97aa5cf4725c04a
+  data.tar.gz: 1147ca7c0ec8a6dc588f68535d4fcd0d59f0d41d6844b87197a7e07a6df7a5a3b716a77be82506c4a3dd35b7b0c5e0443d9c4ac759f27d6644a0d16bbdc9c113

data/config/defaults.yml

@@ -7,3 +7,4 @@ replacements:
   - open data: Taylor Swift
   - opendata: TaylorSwift
 interval: 60 #seconds
+similarity_weighting: 4 # how many words a tweet must overlap already-seen tweets by for us to consider it 'similar'

data/lib/replacer_bot/seen_tweets.rb

@@ -3,13 +3,37 @@ module ReplacerBot
     def self.validate tweet
       archive = retrieve
       t = sanitise tweet
-      valid = not(archive.include? t)
+      valid = not(archive.include? t) && not(similar_to_archive tweet, archive)
       archive.add t
       save archive
 
       valid
     end
 
+    def self.similar_to_archive tweet, archive
+      match = false
+
+      archive.each do |archived_tweet|
+        match = true if similar(tweet, archived_tweet)
+      end
+
+      match
+    end
+
+    def self.similar tweet, other_tweet, weighting: Config.instance.config.similarity_weighting
+      tweet_words = tweet.split ' '
+      return false if tweet_words.count < weighting
+
+      match = false
+
+      (tweet_words.count - (weighting - 1)).times do |i|
+        sample = tweet_words[i, weighting].join(' ').downcase
+        match = true if sanitise(other_tweet.downcase).index sanitise(sample)
+      end
+
+      match
+    end
+
     def self.retrieve
       begin
         Marshal.load File.open Config.instance.config.seen_tweets

data/lib/replacer_bot/version.rb

@@ -1,3 +1,3 @@
 module ReplacerBot
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/spec/lib/replacer_bot/replacer_spec.rb

@@ -60,7 +60,7 @@ module ReplacerBot
 
       it 'filters similar tweets', :vcr do
         SeenTweets.validate 'How open data can help save lives http://t.co/90U7bVq5UF'
-        expect(replacer.tweets.count).to eq 19
+        expect(replacer.tweets.count).to eq 16
       end
     end
 
@@ -75,7 +75,7 @@ module ReplacerBot
       end
 
       it 'actually sends tweets', :vcr do
-        expect(replacer.client).to(receive(:update)).exactly(18).times
+        expect(replacer.client).to(receive(:update)).exactly(16).times
         interval = replacer.config.interval
         replacer.config.interval = 0
         replacer.tweet

data/spec/lib/replacer_bot/seen_tweets_spec.rb

@@ -84,6 +84,30 @@ module ReplacerBot
       ]
     end
 
+    context 'overlap of words' do
+      # n is set in the default config, a lower value makes the bot less noisy at the risk of false negatives
+      it 'does not match on tweets with fewer than n words' do
+        expect(described_class.similar 'appears to match', 'You would think this appears to match').to eq false
+      end
+
+      it 'sees tweets which overlap by at least n words as similar' do
+        expect(described_class.similar 'This is a string of words', 'Also this is a string of words innit').to eq true
+        expect(described_class.similar 'This is a string of words', 'Also this is a similar string similar words innit').to eq false
+        expect(described_class.similar 'This one will be a definite match ', 'So this one will be a definite match no doubt').to eq true
+      end
+
+      it 'deals sensibly with URLs and hashtags' do
+        expect(described_class.similar 'This one has a http://taylor.swift in it', 'So this one has a http://other.url/ in it here').to eq true
+      end
+
+      it 'works on real-world data' do
+        expect(described_class.
+        similar 'Netflix Releases Taylor Swift-Fetching Developer Preview: Netflix has released a developer preview of its in-house… bit.ly/1JfRdgA',
+          'Netflix Releases Taylor Swift-Fetching Developer Preview - Netflix has released a developer preview of its in-house d...'
+        ).to eq true
+      end
+    end
+
     it 'saves a set' do
       set = Set.new [1, 2, 3]
       described_class.save set

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: replacer_bot
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - pikesley