reddit_bot 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f75fa5e061134a60cae44226a5995d8368e802d
-  data.tar.gz: ef7c93b4c345c11d93ab752de2a41ed1d380516b
+  metadata.gz: 68928fa5ea284f33c94d19a8fc4ee0593c3e0ae3
+  data.tar.gz: 67c8d3e031f36c88d2c16e5c2101d0480e30700c
 SHA512:
-  metadata.gz: 1b980165b04f3db528a79a984da5bb21c5d7c4ee9b8e76b15cebb1ce1ef90a81beb68f6af78cb6fb72526686a924078bb4bd95108bfe49906a30158cd836ed82
-  data.tar.gz: 222bd7ef333723ed9b3969b5f27086e864f4a7f1cc021dfdd6d4b057030d26f546466915573db7676717dd140035dffcc264c25a6e6731099db186e0fbe2c3fd
+  metadata.gz: 0f73e132f29c57c0cd11cb6c37b6aa75427467c090c630018bd1a14c98608b38291619bda73d48df0e480764ad93ed3ad2346c672bcc53afbfd0e5e39d63d442
+  data.tar.gz: 5af0b844cf35e5356079121f15496af2e3639855be4a166cd857fb98f82d63a81bcaf3e01a5dfe5dc465e3cd0e074edf327a9da91cf78efe0590bf85b3a935e8

data/Gemfile

@@ -1,3 +1,5 @@
 source "https://rubygems.org"
 
 gemspec
+
+git: "git@github.com:Nakilon/nethttputils.git", tag: "v0.0.1.4"

data/examples/realtimeww2/Gemfile

@@ -1,4 +1,4 @@
 source "https://rubygems.org"
 
 gem "reddit_bot"
-gem "nethttputils", git: "git@github.com:Nakilon/nethttputils.git", tag: "v0.0.1.1"
+gem "nethttputils", git: "git@github.com:Nakilon/nethttputils.git", tag: "v0.0.1.4"

data/examples/realtimeww2/Gemfile.lock

@@ -1,15 +1,15 @@
 GIT
   remote: git@github.com:Nakilon/nethttputils.git
-  revision: 5642bf7ecde4b3ecc55423b9ea9957828a9cb2f9
-  tag: v0.0.1.1
+  revision: 10ae857b46a5db16e0193ca6e53cebbe2c499811
+  tag: v0.0.1.4
   specs:
-    nethttputils (0.0.1.1)
+    nethttputils (0.0.1.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
     json (2.1.0)
-    reddit_bot (1.3.0)
+    reddit_bot (1.3.1)
       json
 
 PLATFORMS
@@ -20,4 +20,4 @@ DEPENDENCIES
   reddit_bot
 
 BUNDLED WITH
-   1.13.1
+   1.15.4

data/examples/realtimeww2/main.rb

@@ -9,37 +9,66 @@ TWITTER_ACCESS_TOKEN = JSON.load(
 
 SUBREDDIT = "RealTimeWW2"
 BOT = RedditBot::Bot.new YAML.load(File.read "secrets.yaml"), subreddit: SUBREDDIT
-
 TWITTER = "RealTimeWWII"
-tweet2text = lambda do |tweet|
+
+tweet2titleNtext = lambda do |tweet|
+  pp tweet if Gem::Platform.local.os == "darwin"
   text = ""
   contains_media = false
   up = ->s{ s.split.map{ |w| "^#{w}" }.join " " }
-  if tweet["entities"]["media"]
+  if tweet["extended_entities"] && tweet["extended_entities"]["media"]
     contains_media = true
-    tweet["entities"]["media"].each_with_index do |media, i|
+    tweet["extended_entities"]["media"].each_with_index do |media, i|
       text.concat "* [Image #{i + 1}](#{media["media_url_https"]})\n\n"
     end
   end
+  if tweet["entities"]["urls"]
+    contains_media = true
+    tweet["entities"]["urls"].each_with_index do |url, i|
+      text.concat "* [Link #{i + 1}](#{url["expanded_url"]})\n\n"
+    end
+  end
   text.concat "^- #{
     up[tweet["user"]["name"]]
   } [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [#{
     up[Date.parse(tweet["created_at"]).strftime "%B %-d, %Y"]
   }](https://twitter.com/#{TWITTER}/status/#{tweet["id"]})"
-  [text, contains_media]
+  require "cgi"
+  [CGI::unescapeHTML(tweet["full_text"]).sub(/( https:\/\/t\.co\/[0-9a-zA-Z]{10})*\z/, ""), text, contains_media]
 end
-test = "* [Image 1](https://pbs.twimg.com/media/DJHq71BXYAA6KJ0.jpg)\n\n^- ^WW2 ^Tweets ^from ^1939 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^September ^7, ^2017](https://twitter.com/#{TWITTER}/status/905764294687633408)"
-unless test == temp = ( tweet2text.call JSON.load NetHTTPUtils.request_data(
-  "https://api.twitter.com/1.1/statuses/show.json?id=905764294687633408&tweet_mode=extended",
-  header: { Authorization: "Bearer #{TWITTER_ACCESS_TOKEN}" }
-) ).first
-  puts "expected:\n#{test.inspect}"
-  puts "got:\n#{temp.inspect}"
-  abort "FORMATTING ERROR"
+[
+  [905764294687633408, "The Polish government & military high command is now evacuating Warsaw for Brest, 120 miles east: German armies are too close to the capital",   "* [Image 1](https://pbs.twimg.com/media/DJHq71BXYAA6KJ0.jpg)\n\n"                                                              "^- ^WW2 ^Tweets ^from ^1939 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^September ^7, ^2017](https://twitter.com/#{TWITTER}/status/905764294687633408)"],
+  [915534673471733760, "In east Poland (now Soviet Ukraine) industry & farms to be collectivised, political parties banned, aristocrats & capitalists \"re-educated\".", "* [Image 1](https://pbs.twimg.com/media/DLSh2J9W4AACcOG.jpg)\n\n* [Image 2](https://pbs.twimg.com/media/DLSh4sKX0AEBaXq.jpg)\n\n^- ^WW2 ^Tweets ^from ^1939 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| ""[^October ^4, ^2017](https://twitter.com/#{TWITTER}/status/915534673471733760)"],
+  [915208866408824832, "For 1st time, RAF planes dropping propaganda leaflets on Berlin itself, entitled \"Germans: these are your leaders!\"",                          "* [Image 1](https://pbs.twimg.com/media/DLN5jJ-XkAEUz9M.jpg)\n\n* [Link 1](https://www.psywar.org/product_1939EH158.php)\n\n"  "^- ^WW2 ^Tweets ^from ^1939 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| ""[^October ^3, ^2017](https://twitter.com/#{TWITTER}/status/915208866408824832)"],
+  [914577848891006978, "\"In Poland, Russia pursued a cold policy of selfinterest. But clearly necessary for Russia… against Nazi menace.\"",                            "* [Link 1](https://www.youtube.com/watch?v=ygmP5A3n2JA)\n\n"                                                                   "^- ^WW2 ^Tweets ^from ^1939 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| ""[^October ^1, ^2017](https://twitter.com/#{TWITTER}/status/914577848891006978)"],
+].each do |id, title_, text_|
+  title, text, contains_media = tweet2titleNtext[ JSON.load NetHTTPUtils.request_data(
+    "https://api.twitter.com/1.1/statuses/show.json?id=#{id}&tweet_mode=extended",
+    header: { Authorization: "Bearer #{TWITTER_ACCESS_TOKEN}" }
+  ) ]
+  unless title_ == title
+    puts "expected:\n#{title_.inspect}"
+    puts "got:\n#{title.inspect}"
+    abort "TITLE FORMATTING ERROR"
+  end
+  unless text_ == text
+    puts "expected:\n#{text_.inspect}"
+    puts "got:\n#{text.inspect}"
+    abort "TEXT FORMATTING ERROR"
+  end
+  if ENV["TEST_POST"]
+    pp BOT.json :post, "/api/submit", {
+      sr: "RealTimeWW2_TEST",
+      kind: "self",
+      title: title,
+      text: text,
+    }.tap{ |h| h.merge!({ flair_id: BOT.json(:get, "/r/RealTimeWW2_TEST/api/link_flair").find{ |flair|
+      flair["text"] == "Contains Media"
+    }["id"] }) if contains_media }
+  end
 end
 abort "OK" if ENV["TEST"]
 
-require "cgi"
 loop do
   id = BOT.new_posts.find do |post|
     /\(https:\/\/twitter\.com\/#{TWITTER}\/status\/(\d{18,})\)/i =~ post["selftext"] and break $1
@@ -63,16 +92,18 @@ loop do
   end ).reverse_each do |tweet|
     next if tweet["id"] <= id
     # next unless tweet["id"] == 905724018996772865    # two media files
-    # tweet["entities"]["urls"].first["url"],
-    text, contains_media = tweet2text[tweet]
+    title, text, contains_media = tweet2titleNtext[tweet]
     result = BOT.json :post, "/api/submit", {
       sr: SUBREDDIT,
       kind: "self",
-      title: CGI::unescapeHTML(tweet["full_text"]).sub(/ https:\/\/t\.co\/[0-9a-zA-Z]{10}\z/, ""),
+      title: title,
       text: text,
     }.tap{ |h| h.merge!({ flair_id: flair["id"] }) if contains_media }
     pp result
-    next if result["json"]["errors"].empty?
+    if result["json"]["errors"].empty?
+      abort if ENV["ONCE"]
+      next
+    end
     fail unless result["json"]["errors"].map(&:first) == ["ALREADY_SUB"]
     puts "ALREADY_SUB error for #{tweet["id"]}"
   end

data/lib/reddit_bot.rb

@@ -5,6 +5,8 @@ require "net/http"
 require "openssl"
 require "json"
 
+require "nethttputils"
+
 require_relative "reddit_bot/version"
 module RedditBot
   class Bot
@@ -96,7 +98,7 @@ module RedditBot
     end
 
     # :yields: JSON objects: ["data"] part of post or self.post
-    def new_posts caching = false
+    def new_posts subreddit = nil, caching = false
       cache = lambda do |id, &block|
         next block.call unless caching
         require "fileutils"
@@ -110,7 +112,7 @@ module RedditBot
       Enumerator.new do |e|
         after = {}
         loop do
-          args = [:get, "/r/#{@subreddit}/new", {limit: 100}.merge(after)]
+          args = [:get, "/#{subreddit || @subreddit || fail}/new", {limit: 100}.merge(after)]
           result = cache.call(args){ json *args }
           fail if result.keys != %w{ kind data }
           fail if result["kind"] != "Listing"
@@ -186,6 +188,7 @@ module RedditBot
     # end
 
     def resp_with_token mtd, path, form
+      fail unless path.start_with? ?/
       nil until _ = catch(:"401") do
         reddit_resp mtd, "https://oauth.reddit.com" + path, form, {
           "Authorization" => "bearer #{token}",
@@ -233,35 +236,7 @@ module RedditBot
     end
 
     def _resp mtd, url, form, headers, base_auth = nil
-      uri = URI.parse url
-      request = if mtd == :get
-        uri.query = URI.encode_www_form form # wtf OpenSSL::SSL::SSLError
-        Net::HTTP::Get.new(uri)
-      else
-        Net::HTTP::Post.new(uri).tap{ |r| r.set_form_data form }
-      end
-      request.basic_auth *base_auth if base_auth
-      headers.each{ |k, v| request[k] = v }
-      # puts request.path
-      # pp request.to_hash
-      # puts request.body
-      http = begin
-        Net::HTTP.start uri.host,
-          use_ssl: uri.scheme == "https",
-          verify_mode: OpenSSL::SSL::VERIFY_NONE,
-          open_timeout: 300
-      rescue OpenSSL::SSL::SSLError, SocketError, Errno::ECONNRESET, Net::OpenTimeout, Errno::ETIMEDOUT, Errno::EPIPE => e
-        puts "ERROR: #{e.class}: #{e}"
-        sleep 5
-        retry
-      end
-      response = begin
-        http.request request
-      rescue OpenSSL::SSL::SSLError, SocketError, Net::ReadTimeout, Errno::EPIPE, EOFError, Zlib::BufError, Errno::ECONNRESET => e
-        puts "ERROR: #{e}"
-        sleep 5
-        retry
-      end
+      response = NetHTTPUtils.get_response url, mtd, form: form, header: headers, auth: base_auth
       puts %w{
         x-ratelimit-remaining
         x-ratelimit-used
@@ -275,13 +250,6 @@ module RedditBot
       fail response.to_hash["x-ratelimit-remaining"][0] \
       if response.to_hash["x-ratelimit-remaining"] &&
          response.to_hash["x-ratelimit-remaining"][0].size < 5
-
-      # if response.code == "401"
-      #   puts request.path
-      #   puts request.body
-      #   pp request.to_hash
-      # end
-
       response
     end
 

data/lib/reddit_bot/version.rb

@@ -1,3 +1,3 @@
 module RedditBot
-  VERSION = "1.3.1" # :nodoc:
+  VERSION = "1.4.0" # :nodoc:
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: reddit_bot
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Victor Maslov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-30 00:00:00.000000000 Z
+date: 2017-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json