twitterscraper-ruby 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2429cf6172b5f19caede64ac35f5c796a7c8a67e76fff8dd2f08677fb15406b
-  data.tar.gz: 0f32ca6b559a18c4e3aac3205f6503149e372d4d7d1976b1e83db26036d9ff17
+  metadata.gz: f4382801b03a5384095aad6a955caea438787fa2eed96e3e001237df368925a2
+  data.tar.gz: 6722b4edce7242b3006e5c097dd78847f36e2da7edea009e2d7b89b09f5b25ff
 SHA512:
-  metadata.gz: a36ce6c91a363b64b36deeb3abbaaaebb725f3449f280b70be92532497a94dc5915ba449926acfacfc0d852d52471d258d41140a8891e64b6040bf262d0c347f
-  data.tar.gz: a737c7db151190a1493b1a2a92bea304cfcf7512b2ee03fc13c6f25794f5dc727fe548e52cb39eccc2a63261fee0d58fc005920a0e7cd7650d20600e184d79cb
+  metadata.gz: 4ca72a0bbce553c38061e0362f755a5e82b47a5288108508410c19a7eef9a2514b58682e88ed1bf89654d5b89c84c41edd8a5fa34fd7d1e5fbf92b267402884a
+  data.tar.gz: 8853b015cb37180d6814710d971a757d08aa4ddd4579af4131e204e34bb10c80ef3139c082f17be92303d9efc2e3f8eb4ba0d15bdf4f264fb4fba0cf87ed42d7

data/.gitignore

@@ -6,5 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
-
+/cache
 /.idea

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.10.0)
+    twitterscraper-ruby (0.11.0)
       nokogiri
       parallel
 

data/README.md

@@ -33,7 +33,7 @@ Command-line interface:
 
 ```shell script
 $ twitterscraper --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
-      --limit 100 --threads 10 --proxy --output output.json
+      --limit 100 --threads 10 --proxy --cache --output output.json
 ```
 
 From Within Ruby:
@@ -143,8 +143,10 @@ $ cat tweets.json | jq . | less
 | `--limit` | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
 | `--threads` | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
 | `--proxy` | Scrape https://twitter.com/search via proxies. | false |
+| `--cache` | Enable caching. | false |
 | `--format` | The format of the output. | json |
 | `--output` | The name of the output file. | tweets.json |
+| `--verbose` | Print debug messages. | tweets.json |
 
 
 ## Contributing

data/bin/twitterscraper

@@ -7,7 +7,7 @@ begin
   cli.parse
   cli.run
 rescue => e
-  STDERR.puts e.message
+  STDERR.puts e.inspect
   STDERR.puts e.backtrace.join("\n")
   exit 1
 end

data/lib/twitterscraper.rb

@@ -2,6 +2,7 @@ require 'twitterscraper/logger'
 require 'twitterscraper/proxy'
 require 'twitterscraper/http'
 require 'twitterscraper/lang'
+require 'twitterscraper/cache'
 require 'twitterscraper/query'
 require 'twitterscraper/client'
 require 'twitterscraper/tweet'

data/lib/twitterscraper/cache.rb

@@ -0,0 +1,69 @@
+require 'base64'
+require 'digest/md5'
+
+module Twitterscraper
+  class Cache
+    def initialize()
+      @ttl = 3600 # 1 hour
+      @dir = 'cache'
+      Dir.mkdir(@dir) unless File.exist?(@dir)
+    end
+
+    def read(key)
+      key = cache_key(key)
+      file = File.join(@dir, key)
+      entry = Entry.from_json(File.read(file))
+      entry.value if entry.time > Time.now - @ttl
+    rescue Errno::ENOENT => e
+      nil
+    end
+
+    def write(key, value)
+      key = cache_key(key)
+      entry = Entry.new(key, value, Time.now)
+      file = File.join(@dir, key)
+      File.write(file, entry.to_json)
+    end
+
+    def fetch(key, &block)
+      if (value = read(key))
+        value
+      else
+        yield.tap { |v| write(key, v) }
+      end
+    end
+
+    def cache_key(key)
+      value = key.gsub(':', '%3A').gsub('/', '%2F').gsub('?', '%3F').gsub('=', '%3D').gsub('&', '%26')
+      value = Digest::MD5.hexdigest(value) if value.length >= 100
+      value
+    end
+
+    class Entry < Hash
+      attr_reader :key, :value, :time
+
+      def initialize(key, value, time)
+        @key = key
+        @value = value
+        @time = time
+      end
+
+      def attrs
+        {key: @key, value: @value, time: @time}
+      end
+
+      def to_json
+        hash = attrs
+        hash[:value] = Base64.encode64(hash[:value])
+        hash.to_json
+      end
+
+      class << self
+        def from_json(text)
+          json = JSON.parse(text)
+          new(json['key'], Base64.decode64(json['value']), Time.parse(json['time']))
+        end
+      end
+    end
+  end
+end

data/lib/twitterscraper/cli.rb

@@ -23,7 +23,7 @@ module Twitterscraper
           threads: options['threads'],
           proxy: options['proxy']
       }
-      client = Twitterscraper::Client.new
+      client = Twitterscraper::Client.new(cache: options['cache'])
       tweets = client.query_tweets(options['query'], query_options)
       export(tweets) unless tweets.empty?
     end
@@ -66,6 +66,7 @@ module Twitterscraper
           'threads:',
           'output:',
           'format:',
+          'cache',
           'proxy',
           'pretty',
           'verbose',

data/lib/twitterscraper/client.rb

@@ -1,5 +1,13 @@
 module Twitterscraper
   class Client
     include Query
+
+    def initialize(cache:)
+      @cache = cache
+    end
+
+    def cache_enabled?
+      @cache
+    end
   end
 end

data/lib/twitterscraper/query.rb

@@ -75,9 +75,22 @@ module Twitterscraper
       query = ERB::Util.url_encode(query)
 
       url = build_query_url(query, lang, pos, from_user)
-      logger.debug("Scraping tweets from #{url}")
+      http_request = lambda do
+        logger.debug("Scraping tweets from #{url}")
+        get_single_page(url, headers, proxies)
+      end
 
-      response = get_single_page(url, headers, proxies)
+      if cache_enabled?
+        client = Cache.new
+        if (response = client.read(url))
+          logger.debug('Fetching tweets from cache')
+        else
+          response = http_request.call
+          client.write(url, response)
+        end
+      else
+        response = http_request.call
+      end
       return [], nil if response.nil?
 
       html, json_resp = parse_single_page(response, pos.nil?)

data/lib/twitterscraper/tweet.rb

@@ -43,6 +43,14 @@ module Twitterscraper
     end
 
     class << self
+      def from_json(text)
+        json = JSON.parse(text)
+        json.map do |tweet|
+          tweet['created_at'] = Time.parse(tweet['created_at'])
+          new(tweet)
+        end
+      end
+
       def from_html(text)
         html = Nokogiri::HTML(text)
         from_tweets_html(html.xpath("//li[@class[contains(., 'js-stream-item')]]/div[@class[contains(., 'js-stream-tweet')]]"))

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-13 00:00:00.000000000 Z
+date: 2020-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -61,6 +61,7 @@ files:
 - bin/twitterscraper
 - lib/twitterscraper-ruby.rb
 - lib/twitterscraper.rb
+- lib/twitterscraper/cache.rb
 - lib/twitterscraper/cli.rb
 - lib/twitterscraper/client.rb
 - lib/twitterscraper/http.rb