twitterscraper-ruby 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf902c947e866cc99e79fbb9f8a51c829accd44aed03ef7657562bf41932c73d
-  data.tar.gz: 1bc5a0698a17b244ee9228d7728767dd00218179a5a49e0852a74cc722322ef0
+  metadata.gz: a950fb24329aaa1020441e258a8a2144100d732142b6c227bb9b026b8bb73996
+  data.tar.gz: 1f64f31e43189e2ee439f5ef6f6d54bc6ea58895adbed67cb8ddbe91af07681a
 SHA512:
-  metadata.gz: 629de8698af1391c210b496e9aadb51ad5f9d7157b1be5d0aa669ae821671e2b5624ba51083fb14b61f93618ff3e90aea1ac0eccb6ea00360fac48a2dfc436c7
-  data.tar.gz: 3f3706bee5f2a92a2addae034201e2e8cee3fef43efdc323be963cbaf1b94c31c53aa49a19e58a068498722dfe07e9796e097fb04364a9afda56d06132e6b935
+  metadata.gz: 8573affbc9a5faa05e5e489364bb2ba0da1aa4f12af35445e5de8b1f8c399eb0575cc9f408b2ba96c3d7fd8b2a74b7dd703229053a33c1f8a883856818033cb9
+  data.tar.gz: 2b2b3ad0b2dd9d089a7b6127ed1b0db21e7f4fa5f0c31e6b366d9b5ae444e2244d4200c813b7a3257f43702d2caa9f264515e701602c24f4482a746b89d41328

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.14.0)
+    twitterscraper-ruby (0.15.0)
       nokogiri
       parallel
 

data/README.md

@@ -33,26 +33,39 @@ $ gem install twitterscraper-ruby
 Command-line interface:
 
 ```shell script
-$ twitterscraper --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
-      --limit 100 --threads 10 --output output.json
+# Returns a collection of relevant tweets matching a specified query.
+$ twitterscraper --type search --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
+      --limit 100 --threads 10 --output tweets.json
+```
+
+```shell script
+# Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+$ twitterscraper --type user --query SCREEN_NAME --limit 100 --output tweets.json
 ```
 
 From Within Ruby:
 
 ```ruby
 require 'twitterscraper'
+client = Twitterscraper::Client.new(cache: true, proxy: true)
+```
 
-options = {
-  start_date: '2020-06-01',
-  end_date:   '2020-06-30',
-  lang:       'ja',
-  limit:      100,
-  threads:    10,
-}
+```ruby
+# Returns a collection of relevant tweets matching a specified query.
+tweets = client.search(KEYWORD, start_date: '2020-06-01', end_date: '2020-06-30', lang: 'ja', limit: 100, threads: 10)
+```
+
+```ruby
+# Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+tweets = client.user_timeline(SCREEN_NAME, limit: 100)
+```
 
-client = Twitterscraper::Client.new(cache: true, proxy: true)
-tweets = client.query_tweets(KEYWORD, options)
 
+## Attributes
+
+### Tweet
+
+```ruby
 tweets.each do |tweet|
   puts tweet.tweet_id
   puts tweet.text
@@ -64,11 +77,6 @@ tweets.each do |tweet|
 end
 ```
 
-
-## Attributes
-
-### Tweet
-
 - screen_name
 - name
 - user_id
@@ -136,6 +144,7 @@ $ cat tweets.json | jq . | less
 | Option | Description | Default |
 | ------------- | ------------- | ------------- |
 | `-h`, `--help` | This option displays a summary of twitterscraper. | |
+| `--type` | Specify a search type. | search |
 | `--query` | Specify a keyword used during the search. | |
 | `--start_date` | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
 | `--end_date` | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |

data/lib/twitterscraper/cli.rb

@@ -16,6 +16,7 @@ module Twitterscraper
       print_version || return if print_version?
 
       query_options = {
+          type: options['type'],
           start_date: options['start_date'],
           end_date: options['end_date'],
           lang: options['lang'],
@@ -59,6 +60,7 @@ module Twitterscraper
           'help',
           'v',
           'version',
+          'type:',
           'query:',
           'start_date:',
           'end_date:',
@@ -75,6 +77,7 @@ module Twitterscraper
           'verbose',
       )
 
+      options['type'] ||= 'search'
       options['start_date'] = Query::OLDEST_DATE if options['start_date'] == 'oldest'
       options['lang'] ||= ''
       options['limit'] = (options['limit'] || 100).to_i

data/lib/twitterscraper/query.rb

@@ -22,23 +22,24 @@ module Twitterscraper
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
         'default&include_available_features=1&include_entities=1&' +
         'reset_error_state=false&src=typd&max_position=__POS__&q=__QUERY__&l=__LANG__'
-    INIT_URL_USER = 'https://twitter.com/{u}'
-    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/{u}/timeline/tweets?' +
+    INIT_URL_USER = 'https://twitter.com/__USER__'
+    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/__USER__/timeline/tweets?' +
         'include_available_features=1&include_entities=1&' +
-        'max_position={pos}&reset_error_state=false'
-
-    def build_query_url(query, lang, pos, from_user = false)
-      # if from_user
-      #   if !pos
-      #     INIT_URL_USER.format(u = query)
-      #   else
-      #     RELOAD_URL_USER.format(u = query, pos = pos)
-      #   end
-      # end
-      if pos
-        RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        'max_position=__POS__&reset_error_state=false'
+
+    def build_query_url(query, lang, from_user, pos)
+      if from_user
+        if pos
+          RELOAD_URL_USER.sub('__USER__', query).sub('__POS__', pos.to_s)
+        else
+          INIT_URL_USER.sub('__USER__', query)
+        end
       else
-        INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        if pos
+          RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        else
+          INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        end
       end
     end
 
@@ -74,11 +75,11 @@ module Twitterscraper
       [items_html, json_resp]
     end
 
-    def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
+    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
       logger.info "Querying #{query}"
       query = ERB::Util.url_encode(query)
 
-      url = build_query_url(query, lang, pos, from_user)
+      url = build_query_url(query, lang, type == 'user', pos)
       http_request = lambda do
         logger.debug "Scraping tweets from #{url}"
         get_single_page(url, headers, proxies)
@@ -107,8 +108,8 @@ module Twitterscraper
 
       if json_resp
         [tweets, json_resp['min_position']]
-      elsif from_user
-        raise NotImplementedError
+      elsif type
+        [tweets, tweets[-1].tweet_id]
       else
         [tweets, "TWEET-#{tweets[-1].tweet_id}-#{tweets[0].tweet_id}"]
       end
@@ -116,7 +117,7 @@ module Twitterscraper
 
     OLDEST_DATE = Date.parse('2006-03-21')
 
-    def validate_options!(queries, start_date:, end_date:, lang:, limit:, threads:)
+    def validate_options!(queries, type:, start_date:, end_date:, lang:, limit:, threads:)
       query = queries[0]
       if query.nil? || query == ''
         raise Error.new('Please specify a search query.')
@@ -161,12 +162,12 @@ module Twitterscraper
       end
     end
 
-    def main_loop(query, lang, limit, daily_limit, headers, proxies)
+    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
       pos = nil
       daily_tweets = []
 
       while true
-        new_tweets, new_pos = query_single_page(query, lang, pos, headers: headers, proxies: proxies)
+        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
         unless new_tweets.empty?
           daily_tweets.concat(new_tweets)
           daily_tweets.uniq! { |t| t.tweet_id }
@@ -195,7 +196,7 @@ module Twitterscraper
       @stop_requested
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
+    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 2)
       start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
       end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
       queries = build_queries(query, start_date, end_date)
@@ -213,7 +214,7 @@ module Twitterscraper
       logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
 
 
-      validate_options!(queries, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
+      validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
       logger.info "The number of threads #{threads}"
 
@@ -229,17 +230,25 @@ module Twitterscraper
         logger.debug "Set 'Thread.abort_on_exception' to true"
 
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           break if stop_requested?
         end
       end
 
       @all_tweets.sort_by { |tweet| (order == 'desc' ? -1 : 1) * tweet.created_at.to_i }
     end
+
+    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
+      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads)
+    end
+
+    def user_timeline(screen_name, limit: 100, order: 'desc')
+      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1)
+    end
   end
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.14.0'
+  VERSION = '0.15.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-16 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri