replacer_bot 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1e8d3be70c7dfae8451a8a7e6655bb74bb545a9
-  data.tar.gz: 062f05a23f866b7eb168f95160b7dc5d07a4632d
+  metadata.gz: 19abcdfabf6c0bf9b60737242db4f0b2b81cfd8a
+  data.tar.gz: 71c81bee80f3775d423de810c51f2473c9ae3139
 SHA512:
-  metadata.gz: 830a1b44dfc123d1ea276ef70deecd5f80276dd9900f43bc6d6c3d7024f87387b0b2acedd26197b028ba8d83509438bb354366bd555219fcf97aa5cf4725c04a
-  data.tar.gz: 1147ca7c0ec8a6dc588f68535d4fcd0d59f0d41d6844b87197a7e07a6df7a5a3b716a77be82506c4a3dd35b7b0c5e0443d9c4ac759f27d6644a0d16bbdc9c113
+  metadata.gz: fbe3eb39215c8e41c236cc10132af163def7147cdaaf20854244f258e0d2dcdef2e08fdda0cde87968c2069d57917c8abfc20c48ae90442853055287065e5eac
+  data.tar.gz: 278f691140d7b6824153d6da70daa72d901358b0fe533e92c20f3cd177b9b4e6e33f1d27ad300903b731f666e3fdb2af79a042eba072b5eed737e33d8ea18d11

data/README.md

@@ -13,6 +13,7 @@ Twitter bot that:
 * Searches Twitter for a phrase
 * Search-and-replaces phrases in the tweets
 * Tweets them
+* Makes a note of the last tweet found so it knows where to start from next time
 
 ## Installation
 
@@ -34,6 +35,7 @@ The default config is [here](https://github.com/pikesley/replacer_bot/blob/maste
       - david cameron: "Satan's Little Helper"
       - cameron: Satan
     save_file: /Users/sam/.replacer_bot/last.tweet
+    seen_tweets: /Users/sam/.replacer_bot/seen.tweets
 
 Notes:
 
@@ -67,3 +69,20 @@ There's also
     ➔ replacer dry_run
 
 which does the search and shows what it would have tweeted, without actually tweeting anything
+
+## Reducing the noise
+
+It turns out that a lot of Twitter is people (or bots) retweeting the same stuff with minimal changes, like adding extra hashtags or using a different URL shortener (I don't really understand how this even happens, but whatever). (Actually, I wonder how much of Twitter is just bots yelling at each other in the void. But I digress.) This makes a crude 'search for this phrase' bot _extremely_ noisy, so I have come up with some Opinions based on some very crude Reckons. Things that will make the bot consider tweets to be 'the same' as tweets we've seen before, and therefore ignorable, are:
+
+* They match save for and URLs they contain being different
+* They match save for different hashtags at the _start_ and _end_ of the tweet (hashtags in the body of the tweet appear to be more relevant, based on my Reckons)
+
+The above reduced the noise a bit, but not enough to make a substantial difference. So I came up with this:
+
+* If there is an overlap of 4 consecutive words between this tweet and one we've seen before, we ignore it
+
+The 4 words thing is tunable in `config.yml`:
+
+    similarity_weighting: 4
+
+but 4 seems about right for my current use case; it will clearly depend on the popularity of your search term

data/lib/replacer_bot.rb

@@ -4,6 +4,7 @@ require 'uri'
 require 'singleton'
 require 'thor'
 require 'yaml'
+require 'htmlentities'
 
 require 'replacer_bot/version'
 require 'replacer_bot/replacer'

data/lib/replacer_bot/helpers.rb

@@ -7,6 +7,11 @@ module ReplacerBot
     word[0] == '#'
   end
 
+  def self.encode_entities string
+    coder = HTMLEntities.new
+    coder.decode string
+  end
+
   def self.last_tweet
     begin
       Marshal.load File.read Config.instance.config.save_file

data/lib/replacer_bot/replacer.rb

@@ -15,7 +15,7 @@ module ReplacerBot
     end
 
     def tweets
-      search.map { |r| ReplacerBot.truncate ReplacerBot.replace string: r.text }
+      search.map { |r| ReplacerBot.truncate ReplacerBot.encode_entities ReplacerBot.replace string: r.text }
     end
 
     def tweet dry_run: false, chatty: false

data/lib/replacer_bot/version.rb

@@ -1,3 +1,3 @@
 module ReplacerBot
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/replacer_bot.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'twitter', '~> 5.14'
   spec.add_dependency 'dotenv', '~> 2.0'
   spec.add_dependency 'thor', '~> 0.19'
+  spec.add_dependency 'htmlentities', '~> 4.3'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/spec/lib/replacer_bot/helpers_spec.rb

@@ -5,6 +5,17 @@ module ReplacerBot
       FileUtils.rm_f Config.instance.config.seen_tweets
     end
 
+    it 'recognises a hashtag' do
+      expect(ReplacerBot.is_hashtag '#hashtag').to eq true
+      expect(ReplacerBot.is_hashtag 'not_hashtag').to eq false
+    end
+
+    it 'encodes HTML entities' do
+      expect(ReplacerBot.encode_entities 'This has a & in it').to eq 'This has a & in it'
+      expect(ReplacerBot.encode_entities 'This has no entities that need coding & ting').
+        to eq 'This has no entities that need coding & ting'
+    end
+
     context 'URLs' do
       it 'URL-encodes a search term' do
         expect(ReplacerBot.encode term: 'open data').to eq '%22open%20data%22'
@@ -88,11 +99,6 @@ module ReplacerBot
         expect(ReplacerBot.replace string: 'This is an Open Data tweet').to eq 'This is a Taylor Swift tweet'
         expect(ReplacerBot.replace string: 'This is an Open Data tweet about an #opendata story').to eq 'This is a Taylor Swift tweet about a #TaylorSwift story'
       end
-
-      it 'recognises a hashtag' do
-        expect(ReplacerBot.is_hashtag '#hashtag').to eq true
-        expect(ReplacerBot.is_hashtag 'not_hashtag').to eq false
-      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: replacer_bot
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - pikesley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-18 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement