wikipedia_twitterbot 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1330f2a58c111f7cee263a8746cd241424ce78cd
-  data.tar.gz: 4ba11d3e6b833f66de96f57bb77150af90216f74
+SHA256:
+  metadata.gz: 35215d526e53248f263a20c08f6b737fdb09ea6df5b814543f9d6a55f19d69cd
+  data.tar.gz: fc3db6117288c4cb6531dc988278bfa39c1990ca598b7d583391c6607138ef7d
 SHA512:
-  metadata.gz: 87da8c83217ea2f27a2150cdbfd9ca4505daf0493e18d96a61fadcadc58b8bf038d954ba0c64018fba1c2a308a8afb2909ff5fd5c8abc6551f21549d21386881
-  data.tar.gz: f82241415cfa16ec0f9c8c1670583e5ce8a2defbe85f00491f1f4b25945cf4192d0dc8a837f52f2474f841cae82fc451289e987e3eb3ac3698ed0ef2231e1503
+  metadata.gz: fce11b334834aa0d8d3f296edd55082d2f3c32bf70dbabc0271eeb09a339862cfec4c95adf382707d5fd790d7224d18b4fb84e3c79709ae57aa42e395d64c578
+  data.tar.gz: aa7dc3aac490003622aad8680847af3e0ffd9500bba5dd8bd20ab28890b09a64c7cf17a047e8bcc11699bd0fd8ae83d03664edb43866d435553cd694c9456932

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+twitter.yml

data/lib/wikipedia_twitterbot/article.rb

@@ -2,9 +2,11 @@ require 'active_record'
 require 'activerecord-import'
 require 'sqlite3'
 require 'logger'
+require 'fileutils'
 require_relative 'tweet'
 require_relative 'twitter_client'
 require_relative 'find_images'
+require_relative 'article_text_cleaner'
 
 class Article < ActiveRecord::Base
   class << self
@@ -88,11 +90,12 @@ class Article < ActiveRecord::Base
   ####################
   # Instance methods #
   ####################
-  def tweet(tweet_text)
-    Tweet.new(tweet_text, filename: @image)
+  def tweet(tweet_text, opts = {})
+    @tweet_result = Tweet.new(tweet_text, opts).result
     self.tweeted = true
     save
-    'tweeted'
+    pp 'tweeted'
+    @tweet_result
   rescue StandardError => e
     self.failed_tweet_at = Time.now
     save
@@ -100,6 +103,7 @@ class Article < ActiveRecord::Base
   end
 
   def screenshot_path
+    FileUtils.mkdir_p('screenshots') unless File.directory?('screenshots')
     "screenshots/#{escaped_title}.png"
   end
 
@@ -134,10 +138,16 @@ class Article < ActiveRecord::Base
     "https://en.wikipedia.org/wiki/#{escaped_title}?veaction=edit&summary=%23#{bot_name}"
   end
 
+  def dirp
+    pp RASTERIZE_PATH
+  end
+
+  RASTERIZE_PATH = "#{__dir__}/rasterize.js".freeze
   def make_screenshot
-    webshot = Webshot::Screenshot.instance
-    webshot.capture mobile_url, "public/#{screenshot_path}",
-                    width: 800, height: 800, allowed_status_codes: [404]
+    # Use rasterize script to make a screenshot
+    %x[phantomjs #{RASTERIZE_PATH} #{mobile_url} #{screenshot_path} 1000px*1000px]
+    # Trim any extra blank space, which may or may not be present.
+    %x[convert #{screenshot_path} -trim #{screenshot_path}]
   end
 
   def hashtag
@@ -148,5 +158,26 @@ class Article < ActiveRecord::Base
     self.class.bot_name
   end
 
+  def wikilinks
+    return @links if @links.present?
+    query = { prop: 'links', titles: title, plnamespace: '0', pllimit: 500 }
+    response = Wiki.query query
+    @links = response.data['pages'].values.first['links'].map { |link| link['title'] }
+    @links
+  end
+
+  def page_text
+    @page_text ||= Wiki.get_page_content title
+  end
+
+  def plaintext
+    @plaintext = ArticleTextCleaner.convert(page_text)
+  end
+
+  def sentence_with(text)
+    # TODO: Remove the plaintext footnote remnants
+    plaintext[/[^.?!\n]*#{Regexp.quote text}[^.?!]*[.?!]/i]
+  end
+
   class NoImageError < StandardError; end
 end

data/lib/wikipedia_twitterbot/article_text_cleaner.rb

@@ -0,0 +1,29 @@
+require 'pandoc-ruby'
+
+class ArticleTextCleaner
+  def self.convert(page_text)
+    new(page_text).convert
+  end
+
+  def initialize(page_text)
+    @page_text = page_text
+  end
+
+  def convert
+    @output = PandocRuby.new(@page_text, from: :mediawiki, to: :plain).convert
+    remove_refs
+    replace_single_linebreaks
+    @output
+  end
+
+  # Refs in up in plaintext as: [12]
+  def remove_refs
+    @output.gsub!(/\[\d+\]/, '')
+  end
+
+  # Linebreaks just for line wrapping appear where spaces should be.
+  # Double line breaks happen between paragraphs; leave those in place.
+  def replace_single_linebreaks
+    @output.gsub!(/(?<!\n)\n(?!\n)/, ' ')
+  end
+end

data/lib/wikipedia_twitterbot/find_articles.rb

@@ -30,10 +30,38 @@ class FindArticles
     by_ids(ids)
   end
 
+  def self.by_title(title)
+    existing = Article.find_by(title: title)
+    return existing if existing.present?
+    page_data = Wiki.query title_info_query(title)
+    article_data = page_data.data['pages'].values.first
+    article = Article.new(id: article_data['pageid'],
+                          title: article_data['title'],
+                          latest_revision: article_data['lastrevid'],
+                          latest_revision_datetime: article_data['touched'])
+
+    return article unless article_data['redirect']
+
+    # If it's a redirect, return the redirect target instead.
+    redirect_target = article.wikilinks.first
+    return by_title(redirect_target)
+  end
+
   ####################
   # Internal methods #
   ####################
 
+  def self.title_revisions_query(title)
+    { prop: 'revisions',
+      titles: title,
+      rvprop: 'userid|ids|timestamp' }
+  end
+
+  def self.title_info_query(title)
+    { prop: 'info',
+      titles: title }
+  end
+
   def self.revisions_query(article_ids)
     { prop: 'revisions',
       pageids: article_ids,

data/lib/wikipedia_twitterbot/find_images.rb

@@ -1,6 +1,5 @@
 class FindImages
   def self.first(article)
-    page_text = Wiki.get_page_content article.title
-    page_text[/File:.{,60}\.jpg/]
+    article.page_text[/File:.{,60}\.jpg/]
   end
 end

data/lib/wikipedia_twitterbot/rasterize.js

@@ -0,0 +1,50 @@
+// Adapted slightly from https://github.com/ariya/phantomjs/blob/master/examples/rasterize.js
+// License: 3-clause BSD https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD
+"use strict";
+var page = require('webpage').create(),
+    system = require('system'),
+    address, output, size, pageWidth, pageHeight;
+
+if (system.args.length < 3 || system.args.length > 5) {
+    console.log('Usage: rasterize.js URL filename [paperwidth*paperheight|paperformat] [zoom]');
+    console.log('  paper (pdf output) examples: "5in*7.5in", "10cm*20cm", "A4", "Letter"');
+    console.log('  image (png/jpg output) examples: "1920px" entire page, window width 1920px');
+    console.log('                                   "800px*600px" window, clipped to 800x600');
+    phantom.exit(1);
+} else {
+    address = system.args[1];
+    output = system.args[2];
+    page.viewportSize = { width: 600, height: 600 };
+    if (system.args.length > 3 && system.args[2].substr(-4) === ".pdf") {
+        size = system.args[3].split('*');
+        page.paperSize = size.length === 2 ? { width: size[0], height: size[1], margin: '0px' }
+                                           : { format: system.args[3], orientation: 'portrait', margin: '1cm' };
+    } else if (system.args.length > 3 && system.args[3].substr(-2) === "px") {
+        size = system.args[3].split('*');
+        if (size.length === 2) {
+            var pageWidth = parseInt(size[0], 10),
+                pageHeight = parseInt(size[1], 10);
+            page.viewportSize = { width: pageWidth, height: pageHeight };
+            page.clipRect = { top: 0, left: 0, width: pageWidth + 20, height: pageHeight };
+        } else {
+            console.log("size:", system.args[3]);
+            var pageWidth = parseInt(system.args[3], 10),
+                pageHeight = parseInt(pageWidth * 3/4, 10); // it's as good an assumption as any
+            console.log ("pageHeight:",pageHeight);
+        }
+    }
+    if (system.args.length > 4) {
+        page.zoomFactor = system.args[4];
+    }
+    page.open(address, function (status) {
+        if (status !== 'success') {
+            console.log('Unable to load the address!');
+            phantom.exit(1);
+        } else {
+            window.setTimeout(function () {
+                page.render(output);
+                phantom.exit();
+            }, 200);
+        }
+    });
+}

data/lib/wikipedia_twitterbot/tweet.rb

@@ -2,6 +2,7 @@ require 'twitter'
 
 # Finds tweetable articles, tweets them
 class Tweet
+  attr_reader :result
   # Find an article to tweet and tweet it
   def self.anything
     # Randomly tweet either the earlier tweetable Article in the database
@@ -20,13 +21,17 @@ class Tweet
   ###############
   # Twitter API #
   ###############
-  def initialize(tweet, filename: nil)
-    if filename
+  def initialize(tweet, opts = {})
+    if opts[:commons_image]
+      filename = opts.delete(:commons_image)
       Wiki.save_commons_image filename
-      TwitterClient.new.client.update_with_media(tweet, File.new(filename))
+      @result = TwitterClient.new.client.update_with_media(tweet, File.new(filename), opts)
       File.delete filename
+    elsif opts[:filename]
+      filename = opts.delete(:filename)
+      @result = TwitterClient.new.client.update_with_media(tweet, File.new(filename), opts)
     else
-      TwitterClient.new.client.update(tweet)
+      @result = TwitterClient.new.client.update(tweet, opts)
     end
   end
 

data/lib/wikipedia_twitterbot/twitter_client.rb

@@ -30,4 +30,8 @@ class TwitterClient
   def hashtags_in(text)
     text.scan(/\s(#\w+)/).flatten
   end
+
+  def trends
+    @client.trends.map(&:name)
+  end
 end

data/lib/wikipedia_twitterbot/version.rb

@@ -1,3 +1,3 @@
 module WikipediaTwitterbot
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/wikipedia_twitterbot.gemspec

@@ -30,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'twitter'
   spec.add_runtime_dependency 'mediawiki_api'
   spec.add_runtime_dependency 'logger'
+  spec.add_runtime_dependency 'pandoc-ruby'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikipedia_twitterbot
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sage Ross
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-27 00:00:00.000000000 Z
+date: 2018-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pandoc-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - sage@ragesoss.com
@@ -139,6 +153,7 @@ files:
 - bin/setup
 - lib/wikipedia_twitterbot.rb
 - lib/wikipedia_twitterbot/article.rb
+- lib/wikipedia_twitterbot/article_text_cleaner.rb
 - lib/wikipedia_twitterbot/category_filter.rb
 - lib/wikipedia_twitterbot/db/001_create_articles.rb
 - lib/wikipedia_twitterbot/db/bootstrap.rb
@@ -147,6 +162,7 @@ files:
 - lib/wikipedia_twitterbot/find_images.rb
 - lib/wikipedia_twitterbot/high_pageviews.rb
 - lib/wikipedia_twitterbot/ores.rb
+- lib/wikipedia_twitterbot/rasterize.js
 - lib/wikipedia_twitterbot/tweet.rb
 - lib/wikipedia_twitterbot/twitter_client.rb
 - lib/wikipedia_twitterbot/version.rb
@@ -173,7 +189,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Tools for building Wikipedia-focused Twitter bots