reddit_bot 1.7.4 → 1.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e081c0ab3c130fb135fa473e6af3fbbc21cfe98
-  data.tar.gz: ef7376ff05f190b9fd8e6b054817de92e4f11b80
+  metadata.gz: 5dca4b66ca047773ac0de806e1281cf7a088bf8f
+  data.tar.gz: d53ad1a90ba44333ab2245b98a304da5e0f89974
 SHA512:
-  metadata.gz: 8638b799d7a414b5a3016a76bc605311abfb27815baf5ec7aea9ead30e241cf523832d747bdb4f5611e82c30a60d4b66bc69463ae5d9a802e1e6c971cf03e48b
-  data.tar.gz: fb618375d7634a1ee42ce47b3e04579a6a15ae028a4eb9523c13ab8294e5c2b7c24e7cbbcb5707119b4e42c952d31edcbcda10a7464f510e046b79bba92ce685
+  metadata.gz: a97e1f8905b56cafa91ce4c3509f7a779db5e5579f8f480363876f45bf4bc152cffd5c4b75ea2040607d95892b831d728981c372440145153577e04ca213e0f6
+  data.tar.gz: 950cbf9d106d01f4f6a68e3a8884d36cc7b0962a4488b3c8363c7d44cda637570e706581b7d07413e0c4d2dfe121f54575b0c14a8bbe3b01176f1f7284a734dc

data/examples/devflairbot/main.rb

@@ -34,7 +34,7 @@ loop do
     JSON.parse( begin
       NetHTTPUtils.request_data "https://www.reddit.com/r/#{subreddit}/comments.json", header: ["User-Agent", "ajsdjasdasd"]
     rescue NetHTTPUtils::Error => e
-      raise unless [503, 504, 500].include? e.code
+      raise unless [500, 502, 503, 504].include? e.code
       sleep 60
       retry
     end )["data"]["children"].each do |comment|

data/examples/largeimages/Gemfile

@@ -2,7 +2,7 @@
 
 gem "json"
 
-gem "nokogiri", "~>1.10.4"
+gem "nokogiri", "~>1.10.8"
 
 gem "nethttputils", git: "git@github.com:nakilon/nethttputils.git"
 gem "directlink", "~>0.0.8.5"

data/examples/largeimages/Gemfile.lock

@@ -76,7 +76,7 @@ GEM
     mini_portile2 (2.4.0)
     multi_json (1.13.1)
     multipart-post (2.0.0)
-    nokogiri (1.10.5)
+    nokogiri (1.10.8)
       mini_portile2 (~> 2.4.0)
     os (0.9.6)
     public_suffix (2.0.5)
@@ -101,7 +101,7 @@ DEPENDENCIES
   google-cloud-error_reporting
   json
   nethttputils!
-  nokogiri (~> 1.10.4)
+  nokogiri (~> 1.10.8)
 
 BUNDLED WITH
    2.0.2

data/examples/largeimages/main.rb

@@ -40,7 +40,7 @@ search_url = lambda do |url|
   JSON.load( begin
     NetHTTPUtils.request_data "https://www.reddit.com/r/largeimages/search.json", form: {q: "url:#{url}", restrict_sr: "on"}, header: ["User-Agent", "ajsdjasdasd"]
   rescue NetHTTPUtils::Error => e
-    raise unless [503].include? e.code
+    raise unless [500, 503].include? e.code
     sleep 60
     retry
   end )["data"]["children"]
@@ -85,13 +85,22 @@ loop do
   ].each do |source, min_resolution, entries|
     logger.warn "#{source}.size: #{entries.size}"
     entries.each do |id, url, title, subreddit, author, permalink|
+      author.downcase!
       next if checked.include? id
       checked << id
       # next if Gem::Platform.local.os == "darwin" # prevent concurrent posting
-      logger.debug "image url for #{id}: #{url}"
-      next logger.warn "skipped a post by /u/sjhill"       if author == "sjhill"        # opt-out
-      next logger.warn "skipped a post by /u/redisforever" if author == "redisforever"  # opt-out
-      next logger.warn "skipped a post by /u/bekalaki"     if author == "bekalaki"      # 9 ways to divide a karmawhore
+      logger.info "image url for #{id}: #{url}"
+      next logger.warn "skipped a post by /u/sjhill"          if author == "sjhill"          # opt-out
+      next logger.warn "skipped a post by /u/redisforever"    if author == "redisforever"    # opt-out
+      next logger.warn "skipped a post by /u/bekalaki"        if author == "bekalaki"        # 9 ways to divide a karmawhore
+      next logger.warn "skipped a post by /u/cherryblackeyes" if author == "cherryblackeyes" # he's not nice
+      next logger.warn "skipped a post by /u/abel_a_kay"      if author == "abel_a_kay"      # posting very similar images of the same thing for the history
+      next logger.warn "skipped gifv" if ( begin
+        URI url
+      rescue URI::InvalidURIError
+        require "addressable"
+        URI Addressable::URI.escape url
+      end ).host.split(?.) == %w{ v redd it }
 
       t = begin
         DirectLink url, 60
@@ -105,11 +114,11 @@ loop do
              DirectLink::ErrorBadLink => e
         next logger.error "skipped (#{e}) #{url} from http://redd.it/#{id}"
       end
-      logger.info "DirectLink: #{t.inspect}"
+      logger.debug "DirectLink: #{t.inspect}"
       tt = t.is_a?(Array) ? t : [t]
       next logger.error "probably crosspost of a self post: http://redd.it/#{id}" if tt.empty?
       unless min_resolution <= tt.first.width * tt.first.height
-        next logger.debug "skipped low resolution #{source}"
+        next logger.info "skipped low resolution #{source}"
       end
       # puts "https://www.reddit.com/r/LargeImages/search.json?q=url%3A#{CGI.escape url}&restrict_sr=on"
       resolution = "[#{tt.first.width}x#{tt.first.height}]"

data/examples/realtimeww2/main.rb

@@ -37,11 +37,11 @@ tweet2titleNtext = lambda do |tweet|
   [CGI::unescapeHTML(tweet["full_text"]).sub(/( https:\/\/t\.co\/[0-9a-zA-Z]{10})*\z/, ""), text, contains_media]
 end
 [
-  [905764294687633408,  true, "The Polish government & military high command is now evacuating Warsaw for Brest, 120 miles east: German armies are too close to the capital",   "* [Image 1](https://pbs.twimg.com/media/DJHq71BXYAA6KJ0.jpg)\n\n"                                                              "^- ^WW2 ^Tweets ^from ^1941 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^""September ^7, ^2017](https://twitter.com/#{TWITTER}/status/905764294687633408)"],
-  [915534673471733760,  true, "In east Poland (now Soviet Ukraine) industry & farms to be collectivised, political parties banned, aristocrats & capitalists \"re-educated\".", "* [Image 1](https://pbs.twimg.com/media/DLSh2J9W4AACcOG.jpg)\n\n* [Image 2](https://pbs.twimg.com/media/DLSh4sKX0AEBaXq.jpg)\n\n^- ^WW2 ^Tweets ^from ^1941 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^4, ^2017](https://twitter.com/#{TWITTER}/status/915534673471733760)"],
-  [915208866408824832,  true, "For 1st time, RAF planes dropping propaganda leaflets on Berlin itself, entitled \"Germans: these are your leaders!\"",                          "* [Image 1](https://pbs.twimg.com/media/DLN5jJ-XkAEUz9M.jpg)\n\n* [Link 1](https://www.psywar.org/product_1939EH158.php)\n\n"  "^- ^WW2 ^Tweets ^from ^1941 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^3, ^2017](https://twitter.com/#{TWITTER}/status/915208866408824832)"],
-  [914577848891006978,  true, "\"In Poland, Russia pursued a cold policy of selfinterest. But clearly necessary for Russia… against Nazi menace.\"",                            "* [Link 1](https://www.youtube.com/watch?v=ygmP5A3n2JA)\n\n"                                                                   "^- ^WW2 ^Tweets ^from ^1941 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^1, ^2017](https://twitter.com/#{TWITTER}/status/914577848891006978)"],
-  [926581977372942336, false, "Finland rejects Soviet demand to surrender land near Leningrad & give Red Navy base in Hanko; Soviets now claim Finns' manner \"warlike\".",                                                                                                                                     "^- ^WW2 ^Tweets ^from ^1941 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^" "November ^3, ^2017](https://twitter.com/#{TWITTER}/status/926581977372942336)"],
+  [905764294687633408,  true, "The Polish government & military high command is now evacuating Warsaw for Brest, 120 miles east: German armies are too close to the capital",   "* [Image 1](https://pbs.twimg.com/media/DJHq71BXYAA6KJ0.jpg)\n\n"                                                              "^- ^WW2 ^Tweets ^from ^1942 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^""September ^7, ^2017](https://twitter.com/#{TWITTER}/status/905764294687633408)"],
+  [915534673471733760,  true, "In east Poland (now Soviet Ukraine) industry & farms to be collectivised, political parties banned, aristocrats & capitalists \"re-educated\".", "* [Image 1](https://pbs.twimg.com/media/DLSh2J9W4AACcOG.jpg)\n\n* [Image 2](https://pbs.twimg.com/media/DLSh4sKX0AEBaXq.jpg)\n\n^- ^WW2 ^Tweets ^from ^1942 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^4, ^2017](https://twitter.com/#{TWITTER}/status/915534673471733760)"],
+  [915208866408824832,  true, "For 1st time, RAF planes dropping propaganda leaflets on Berlin itself, entitled \"Germans: these are your leaders!\"",                          "* [Image 1](https://pbs.twimg.com/media/DLN5jJ-XkAEUz9M.jpg)\n\n* [Link 1](https://www.psywar.org/product_1939EH158.php)\n\n"  "^- ^WW2 ^Tweets ^from ^1942 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^3, ^2017](https://twitter.com/#{TWITTER}/status/915208866408824832)"],
+  [914577848891006978,  true, "\"In Poland, Russia pursued a cold policy of selfinterest. But clearly necessary for Russia… against Nazi menace.\"",                            "* [Link 1](https://www.youtube.com/watch?v=ygmP5A3n2JA)\n\n"                                                                   "^- ^WW2 ^Tweets ^from ^1942 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^"  "October ^1, ^2017](https://twitter.com/#{TWITTER}/status/914577848891006978)"],
+  [926581977372942336, false, "Finland rejects Soviet demand to surrender land near Leningrad & give Red Navy base in Hanko; Soviets now claim Finns' manner \"warlike\".",                                                                                                                                     "^- ^WW2 ^Tweets ^from ^1942 [^\\(@#{TWITTER}\\)](https://twitter.com/#{TWITTER}) ^| [^" "November ^3, ^2017](https://twitter.com/#{TWITTER}/status/926581977372942336)"],
 ].each do |id, contains_media_, title_, text_|
   title, text, contains_media = tweet2titleNtext[ JSON.load NetHTTPUtils.request_data(
     "https://api.twitter.com/1.1/statuses/show.json",
@@ -101,7 +101,7 @@ loop do
       sleep t
     end
   rescue NetHTTPUtils::Error => e
-    fail if e.code != 503
+    fail unless [500, 503].include? e.code
     sleep timeout
     timeout *= 2
     retry

data/examples/wallpaper/Gemfile.lock

@@ -1,25 +1,26 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
-    directlink (0.0.4.7)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    directlink (0.0.8.7)
       addressable
       fastimage (~> 2.1.3)
       kramdown
+      nethttputils (~> 0.3.3.0)
       nokogiri
-      reddit_bot (~> 1.6.13)
-    fastimage (2.1.5)
-    json (2.1.0)
-    kramdown (1.17.0)
+      reddit_bot (~> 1.7.0)
+    fastimage (2.1.7)
+    json (2.3.0)
+    kramdown (2.1.0)
     mini_portile2 (2.4.0)
-    nethttputils (0.2.5.1)
+    nethttputils (0.3.3.0)
     nokogiri (1.10.4)
       mini_portile2 (~> 2.4.0)
     public_suffix (2.0.5)
-    reddit_bot (1.6.13)
+    reddit_bot (1.7.3)
       json
-      nethttputils (~> 0.2.5.1)
+      nethttputils (~> 0.3.3.0)
 
 PLATFORMS
   ruby

data/lib/reddit_bot.rb

@@ -3,6 +3,7 @@ STDOUT.sync = true
 
 require "openssl"
 require "json"
+require "yaml"
 
 require "nethttputils"
 
@@ -234,10 +235,10 @@ module RedditBot
           "User-Agent" => "bot/#{@user_agent || @name}/#{RedditBot::VERSION} by /u/nakilon",
         }
       rescue NetHTTPUtils::Error => e
+        raise unless e.code == 401
         sleep timeout
         Module.nesting[1].logger.info "sleeping #{timeout} seconds because of #{e.code}"
         timeout *= 2
-        raise unless e.code == 401
         @token_cached = nil
         retry
       end
@@ -247,21 +248,7 @@ module RedditBot
       mtd, url, form, headers, basic_auth = *args
       headers["Cookie:"] = "over18=1"
       begin
-        NetHTTPUtils.request_data(url, mtd, form: form, header: headers, auth: basic_auth) do |response|
-          next unless remaining = response.to_hash["x-ratelimit-remaining"]
-          if Gem::Platform.local.os == "darwin"
-            Module.nesting[1].logger.debug %w{
-              x-ratelimit-remaining
-              x-ratelimit-used
-              x-ratelimit-reset
-            }.map{ |key| "#{key}=#{response.to_hash[key]}" }.join ", "
-          end
-          fail remaining[0] if remaining[0].size < 4
-          next if remaining[0].size > 4
-          t = (response.to_hash["x-ratelimit-reset"][0].to_f + 1) / [remaining[0].to_f - 10, 1].max + 1
-          Module.nesting[1].logger.info "sleeping #{t} seconds because of x-ratelimit"
-          sleep t
-        end
+        NetHTTPUtils.request_data url, mtd, form: form, header: headers, auth: basic_auth
       rescue NetHTTPUtils::Error => e
         sleep 5
         raise unless e.code.to_s.start_with? "50"
@@ -271,4 +258,66 @@ module RedditBot
     end
 
   end
+
+  module Twitter
+    require "json"
+
+    def self.init_twitter twitter
+      const_set :TWITTER_ACCOUNT, twitter
+      const_set :TWITTER_ACCESS_TOKEN, JSON.load(
+        NetHTTPUtils.request_data "https://api.twitter.com/oauth2/token", :post,
+          auth: File.read("twitter.token").split,
+          form: {grant_type: :client_credentials}
+      )["access_token"]
+    end
+
+    require "cgi"
+    def self.tweet2titleNtext tweet
+      text = ""
+      contains_media = false
+      up = ->s{ s.split.map{ |w| "^#{w}" }.join " " }
+      if tweet["extended_entities"] && !tweet["extended_entities"]["media"].empty?
+        contains_media = true
+        tweet["extended_entities"]["media"].each_with_index do |media, i|
+          text.concat "* [Image #{i + 1}](#{media["media_url_https"]})\n\n"
+        end
+      end
+      if !tweet["entities"]["urls"].empty?
+        contains_media = true
+        tweet["entities"]["urls"].each_with_index do |url, i|
+          text.concat "* [Link #{i + 1}](#{url["expanded_url"]})\n\n"
+        end
+      end
+      text.concat "^- #{
+        up[tweet["user"]["name"]]
+      } [^\\(@#{TWITTER_ACCOUNT}\\)](https://twitter.com/#{TWITTER_ACCOUNT}) ^| [#{
+        up[Date.parse(tweet["created_at"]).strftime "%B %-d, %Y"]
+      }](https://twitter.com/#{TWITTER_ACCOUNT}/status/#{tweet["id"]})"
+      [CGI::unescapeHTML(tweet["full_text"]).sub(/( https:\/\/t\.co\/[0-9a-zA-Z]{10})*\z/, ""), text, contains_media]
+    end
+
+    def self.user_timeline
+      timeout = 1
+      JSON.load begin
+        NetHTTPUtils.request_data(
+          "https://api.twitter.com/1.1/statuses/user_timeline.json",
+          form: { screen_name: TWITTER_ACCOUNT, count: 200, tweet_mode: "extended" },
+          header: { Authorization: "Bearer #{TWITTER_ACCESS_TOKEN}" }
+        ) do |res|
+          next unless res.key? "x-rate-limit-remaining"
+          remaining = res.fetch("x-rate-limit-remaining").to_i
+          next if 100 < remaining
+          t = (res.fetch("x-rate-limit-reset").to_i - Time.now.to_i + 1).fdiv remaining
+          puts "sleep #{t}"
+          sleep t
+        end
+      rescue NetHTTPUtils::Error => e
+        fail unless [500, 503].include? e.code
+        sleep timeout
+        timeout *= 2
+        retry
+      end
+    end
+  end
+
 end

data/reddit_bot.gemspec

@@ -1,8 +1,6 @@
-require_relative "lib/reddit_bot/version"
-
 Gem::Specification.new do |spec|
   spec.name          = "reddit_bot"
-  spec.version       = RedditBot::VERSION
+  spec.version       = "1.7.5"
   spec.authors       = ["Victor Maslov"]
   spec.email         = ["nakilon@gmail.com"]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: reddit_bot
 version: !ruby/object:Gem::Version
-  version: 1.7.4
+  version: 1.7.5
 platform: ruby
 authors:
 - Victor Maslov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-04 00:00:00.000000000 Z
+date: 2020-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -108,7 +108,6 @@ files:
 - examples/yayornay/Gemfile.lock
 - examples/yayornay/main.rb
 - lib/reddit_bot.rb
-- lib/reddit_bot/version.rb
 - reddit_bot.gemspec
 homepage: https://github.com/Nakilon/reddit_bot
 licenses:

data/lib/reddit_bot/version.rb

@@ -1,3 +0,0 @@
-module RedditBot
-  VERSION = "1.7.4" # :nodoc:
-end