twitterscraper-ruby 0.12.0 → 0.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e98afb0444b724e0c9c29f6b888c017166859d1252337f34686526060ca8368d
-  data.tar.gz: 5ef3ff7f86d9a0c9dd1883d55498049d0f164aa7c71a7c9c2bbf0a89ae9bb32c
+  metadata.gz: 7f7d320841125d9a582ece6083f421f0abf301addbc5c5c2a3d2b2c09bedbc33
+  data.tar.gz: 6ea43165ffa4f37c4319566689a42f2f275d8a70402b0d6b4164df519fee90b5
 SHA512:
-  metadata.gz: 04ef61c57545cbbdbbe5da53d1f24cf064b7d1c61ad3da9bc57a361d24ed24480c4f68fa1fea67345ceff4d4d4685f046a4586f55ebe8f3dc0ca6332c7c2d928
-  data.tar.gz: f5fd19c8289c7caf574dc78f754ba9aaf9446f3819b394d14414909b1505e0f9b25181802448d28285de8db81a27e12e0e65d1e1a0b2b0e5df8e7e73d6263e14
+  metadata.gz: ee3756538ec28e9f0113e611e2731ec33107dabacf7cb730b257d6c94351407ef171a9bc91402a589fa73fdb6b705f73b11582766af1d04a3413b8bc79dc6619
+  data.tar.gz: 78200dc658a9c1cf43ed7367e499b0d1b243728aecb2ffd7366b5612f8905bb33d27ab7e1412327d05b7fff159196fe9e24d18c8cc4c24898af10533fbdf43df

data/.circleci/config.yml

@@ -0,0 +1,31 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2
+
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.6.4-stretch-node
+        environment:
+          BUNDLER_VERSION: 2.1.4
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Update bundler
+          command: gem update bundler
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - restore_cache:
+          keys:
+            - gem-cache-v1-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+            - gem-cache-v1-{{ arch }}-{{ .Branch }}
+            - gem-cache-v1
+      - run: bundle install --path vendor/bundle
+      - run: bundle clean
+      - save_cache:
+          key: gem-cache-v1-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+          paths:
+            - vendor/bundle
+      - run: bundle exec rspec

data/.rspec

@@ -0,0 +1,2 @@
+-fd
+--require spec_helper

data/Gemfile

@@ -5,3 +5,4 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "minitest", "~> 5.0"
+gem "rspec"

data/Gemfile.lock

@@ -1,19 +1,33 @@
 PATH
   remote: .
   specs:
-    twitterscraper-ruby (0.12.0)
+    twitterscraper-ruby (0.15.2)
       nokogiri
       parallel
 
 GEM
   remote: https://rubygems.org/
   specs:
+    diff-lcs (1.4.4)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     parallel (1.19.2)
     rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
 
 PLATFORMS
   ruby
@@ -21,6 +35,7 @@ PLATFORMS
 DEPENDENCIES
   minitest (~> 5.0)
   rake (~> 12.0)
+  rspec
   twitterscraper-ruby!
 
 BUNDLED WITH

data/README.md

@@ -1,18 +1,21 @@
 # twitterscraper-ruby
 
+[![Build Status](https://circleci.com/gh/ts-3156/twitterscraper-ruby.svg?style=svg)](https://circleci.com/gh/ts-3156/twitterscraper-ruby)
 [![Gem Version](https://badge.fury.io/rb/twitterscraper-ruby.svg)](https://badge.fury.io/rb/twitterscraper-ruby)
 
 A gem to scrape https://twitter.com/search. This gem is inspired by [taspinar/twitterscraper](https://github.com/taspinar/twitterscraper).
 
+Please feel free to ask [@ts_3156](https://twitter.com/ts_3156) if you have any questions.
+
 
 ## Twitter Search API vs. twitterscraper-ruby
 
-### Twitter Search API
+#### Twitter Search API
 
 - The number of tweets: 180 - 450 requests/15 minutes (18,000 - 45,000 tweets/15 minutes)
 - The time window: the past 7 days
 
-### twitterscraper-ruby
+#### twitterscraper-ruby
 
 - The number of tweets: Unlimited
 - The time window: from 2006-3-21 to today
@@ -29,45 +32,92 @@ $ gem install twitterscraper-ruby
 
 ## Usage
 
-Command-line interface:
+#### Command-line interface:
+
+Returns a collection of relevant tweets matching a specified query.
 
 ```shell script
-$ twitterscraper --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
-      --limit 100 --threads 10 --proxy --cache --output output.json
+$ twitterscraper --type search --query KEYWORD --start_date 2020-06-01 --end_date 2020-06-30 --lang ja \
+      --limit 100 --threads 10 --output tweets.json
 ```
 
-From Within Ruby:
+Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+
+```shell script
+$ twitterscraper --type user --query SCREEN_NAME --limit 100 --output tweets.json
+```
+
+#### From Within Ruby:
 
 ```ruby
 require 'twitterscraper'
+client = Twitterscraper::Client.new(cache: true, proxy: true)
+```
 
-options = {
-  start_date: '2020-06-01',
-  end_date:   '2020-06-30',
-  lang:       'ja',
-  limit:      100,
-  threads:    10,
-  proxy:      true
-}
+Returns a collection of relevant tweets matching a specified query.
 
-client = Twitterscraper::Client.new
-tweets = client.query_tweets(KEYWORD, options)
+```ruby
+tweets = client.search(KEYWORD, start_date: '2020-06-01', end_date: '2020-06-30', lang: 'ja', limit: 100, threads: 10)
+```
+
+Returns a collection of the most recent tweets posted by the user indicated by the screen_name
+
+```ruby
+tweets = client.user_timeline(SCREEN_NAME, limit: 100)
+```
 
+
+## Examples
+
+```shell script
+$ twitterscraper --query twitter --limit 1000
+$ cat tweets.json | jq . | less
+```
+
+
+## Attributes
+
+### Tweet
+
+```ruby
 tweets.each do |tweet|
   puts tweet.tweet_id
   puts tweet.text
   puts tweet.tweet_url
   puts tweet.created_at
 
+  attr_names = hash.keys
   hash = tweet.attrs
-  puts hash.keys
+  json = tweet.to_json
 end
 ```
 
-
-## Attributes
-
-### Tweet
+```json
+[
+  {
+      "screen_name": "@name",
+      "name": "Name",
+      "user_id": 12340000,
+      "tweet_id": 1234000000000000,
+      "text": "Thanks Twitter!",
+      "links": [],
+      "hashtags": [],
+      "image_urls": [],
+      "video_url": null,
+      "has_media": null,
+      "likes": 10,
+      "retweets": 20,
+      "replies": 0,
+      "is_replied": false,
+      "is_reply_to": false,
+      "parent_tweet_id": null,
+      "reply_to_users": [],
+      "tweet_url": "https://twitter.com/name/status/1234000000000000",
+      "timestamp": 1594793000,
+      "created_at": "2020-07-15 00:00:00 +0000"
+    }
+]
+```
 
 - screen_name
 - name
@@ -110,43 +160,24 @@ end
 Search operators documentation is in [Standard search operators](https://developer.twitter.com/en/docs/tweets/rules-and-filtering/overview/standard-operators).
 
 
-## Examples
-
-```shell script
-$ twitterscraper --query twitter --limit 1000
-$ cat tweets.json | jq . | less
-```
-
-```json
-[
-  {
-    "screen_name": "@screenname",
-    "name": "name",
-    "user_id": 1194529546483000000,
-    "tweet_id": 1282659891992000000,
-    "tweet_url": "https://twitter.com/screenname/status/1282659891992000000",
-    "created_at": "2020-07-13 12:00:00 +0000",
-    "text": "Thanks Twitter!"
-  }
-]
-```
-
 ## CLI Options
 
-| Option | Description | Default |
-| ------------- | ------------- | ------------- |
-| `-h`, `--help` | This option displays a summary of twitterscraper. | |
-| `--query` | Specify a keyword used during the search. | |
-| `--start_date` | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
-| `--end_date` | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |
-| `--lang` | Retrieve tweets written in a specific language. | |
-| `--limit` | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
-| `--threads` | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
-| `--proxy` | Scrape https://twitter.com/search via proxies. | false |
-| `--cache` | Enable caching. | false |
-| `--format` | The format of the output. | json |
-| `--output` | The name of the output file. | tweets.json |
-| `--verbose` | Print debug messages. | tweets.json |
+| Option | Type | Description | Value |
+| ------------- | ------------- | ------------- | ------------- |
+| `--help`       | string  | This option displays a summary of twitterscraper. | |
+| `--type`       | string  | Specify a search type. | search(default) or user |
+| `--query`      | string  | Specify a keyword used during the search. | |
+| `--start_date` | string  | Used as "since:yyyy-mm-dd for your query. This means "since the date". | |
+| `--end_date`   | string  | Used as "until:yyyy-mm-dd for your query. This means "before the date". | |
+| `--lang`       | string  | Retrieve tweets written in a specific language. | |
+| `--limit`      | integer | Stop scraping when *at least* the number of tweets indicated with --limit is scraped. | 100 |
+| `--order`      | string  | Sort a order of the results. | desc(default) or asc |
+| `--threads`    | integer | Set the number of threads twitterscraper-ruby should initiate while scraping for your query. | 2 |
+| `--proxy`      | boolean | Scrape https://twitter.com/search via proxies. | true(default) or false |
+| `--cache`      | boolean | Enable caching. | true(default) or false |
+| `--format`     | string  | The format of the output. | json(default) or html |
+| `--output`     | string  | The name of the output file. | tweets.json |
+| `--verbose`    |         | Print debug messages. | |
 
 
 ## Contributing

data/lib/twitterscraper.rb

@@ -4,6 +4,7 @@ require 'twitterscraper/http'
 require 'twitterscraper/lang'
 require 'twitterscraper/cache'
 require 'twitterscraper/query'
+require 'twitterscraper/type'
 require 'twitterscraper/client'
 require 'twitterscraper/tweet'
 require 'twitterscraper/template'

data/lib/twitterscraper/cli.rb

@@ -16,15 +16,16 @@ module Twitterscraper
       print_version || return if print_version?
 
       query_options = {
+          type: options['type'],
           start_date: options['start_date'],
           end_date: options['end_date'],
           lang: options['lang'],
           limit: options['limit'],
           daily_limit: options['daily_limit'],
+          order: options['order'],
           threads: options['threads'],
-          proxy: options['proxy']
       }
-      client = Twitterscraper::Client.new(cache: options['cache'])
+      client = Twitterscraper::Client.new(cache: options['cache'], proxy: options['proxy'])
       tweets = client.query_tweets(options['query'], query_options)
       export(tweets) unless tweets.empty?
     end
@@ -59,29 +60,36 @@ module Twitterscraper
           'help',
           'v',
           'version',
+          'type:',
           'query:',
           'start_date:',
           'end_date:',
           'lang:',
           'limit:',
           'daily_limit:',
+          'order:',
           'threads:',
           'output:',
           'format:',
-          'cache',
-          'proxy',
+          'cache:',
+          'proxy:',
           'pretty',
           'verbose',
       )
 
+      options['type'] ||= 'search'
       options['start_date'] = Query::OLDEST_DATE if options['start_date'] == 'oldest'
       options['lang'] ||= ''
       options['limit'] = (options['limit'] || 100).to_i
       options['daily_limit'] = options['daily_limit'].to_i if options['daily_limit']
       options['threads'] = (options['threads'] || 2).to_i
       options['format'] ||= 'json'
+      options['order'] ||= 'desc'
       options['output'] ||= "tweets.#{options['format']}"
 
+      options['cache'] = options['cache'] != 'false'
+      options['proxy'] = options['proxy'] != 'false'
+
       options
     end
 

data/lib/twitterscraper/client.rb

@@ -2,12 +2,17 @@ module Twitterscraper
   class Client
     include Query
 
-    def initialize(cache: false)
+    def initialize(cache: true, proxy: true)
       @cache = cache
+      @proxy = proxy
     end
 
     def cache_enabled?
       @cache
     end
+
+    def proxy_enabled?
+      @proxy
+    end
   end
 end

data/lib/twitterscraper/query.rb

@@ -22,23 +22,24 @@ module Twitterscraper
     RELOAD_URL = 'https://twitter.com/i/search/timeline?f=tweets&vertical=' +
         'default&include_available_features=1&include_entities=1&' +
         'reset_error_state=false&src=typd&max_position=__POS__&q=__QUERY__&l=__LANG__'
-    INIT_URL_USER = 'https://twitter.com/{u}'
-    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/{u}/timeline/tweets?' +
+    INIT_URL_USER = 'https://twitter.com/__USER__'
+    RELOAD_URL_USER = 'https://twitter.com/i/profiles/show/__USER__/timeline/tweets?' +
         'include_available_features=1&include_entities=1&' +
-        'max_position={pos}&reset_error_state=false'
-
-    def build_query_url(query, lang, pos, from_user = false)
-      # if from_user
-      #   if !pos
-      #     INIT_URL_USER.format(u = query)
-      #   else
-      #     RELOAD_URL_USER.format(u = query, pos = pos)
-      #   end
-      # end
-      if pos
-        RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        'max_position=__POS__&reset_error_state=false'
+
+    def build_query_url(query, lang, type, pos)
+      if type.user?
+        if pos
+          RELOAD_URL_USER.sub('__USER__', query).sub('__POS__', pos.to_s)
+        else
+          INIT_URL_USER.sub('__USER__', query)
+        end
       else
-        INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        if pos
+          RELOAD_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s).sub('__POS__', pos)
+        else
+          INIT_URL.sub('__QUERY__', query).sub('__LANG__', lang.to_s)
+        end
       end
     end
 
@@ -50,7 +51,7 @@ module Twitterscraper
       end
       Http.get(url, headers, proxy, timeout)
     rescue => e
-      logger.debug "query_single_page: #{e.inspect}"
+      logger.debug "get_single_page: #{e.inspect}"
       if (retries -= 1) > 0
         logger.info "Retrying... (Attempts left: #{retries - 1})"
         retry
@@ -74,11 +75,11 @@ module Twitterscraper
       [items_html, json_resp]
     end
 
-    def query_single_page(query, lang, pos, from_user = false, headers: [], proxies: [])
+    def query_single_page(query, lang, type, pos, headers: [], proxies: [])
       logger.info "Querying #{query}"
       query = ERB::Util.url_encode(query)
 
-      url = build_query_url(query, lang, pos, from_user)
+      url = build_query_url(query, lang, type, pos)
       http_request = lambda do
         logger.debug "Scraping tweets from #{url}"
         get_single_page(url, headers, proxies)
@@ -107,8 +108,8 @@ module Twitterscraper
 
       if json_resp
         [tweets, json_resp['min_position']]
-      elsif from_user
-        raise NotImplementedError
+      elsif type.user?
+        [tweets, tweets[-1].tweet_id]
       else
         [tweets, "TWEET-#{tweets[-1].tweet_id}-#{tweets[0].tweet_id}"]
       end
@@ -116,7 +117,8 @@ module Twitterscraper
 
     OLDEST_DATE = Date.parse('2006-03-21')
 
-    def validate_options!(query, start_date:, end_date:, lang:, limit:, threads:, proxy:)
+    def validate_options!(queries, type:, start_date:, end_date:, lang:, limit:, threads:)
+      query = queries[0]
       if query.nil? || query == ''
         raise Error.new('Please specify a search query.')
       end
@@ -160,12 +162,12 @@ module Twitterscraper
       end
     end
 
-    def main_loop(query, lang, limit, daily_limit, headers, proxies)
+    def main_loop(query, lang, type, limit, daily_limit, headers, proxies)
       pos = nil
       daily_tweets = []
 
       while true
-        new_tweets, new_pos = query_single_page(query, lang, pos, headers: headers, proxies: proxies)
+        new_tweets, new_pos = query_single_page(query, lang, type, pos, headers: headers, proxies: proxies)
         unless new_tweets.empty?
           daily_tweets.concat(new_tweets)
           daily_tweets.uniq! { |t| t.tweet_id }
@@ -194,16 +196,27 @@ module Twitterscraper
       @stop_requested
     end
 
-    def query_tweets(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, threads: 2, proxy: false)
+    def query_tweets(query, type: 'search', start_date: nil, end_date: nil, lang: nil, limit: 100, daily_limit: nil, order: 'desc', threads: 2)
       start_date = Date.parse(start_date) if start_date && start_date.is_a?(String)
       end_date = Date.parse(end_date) if end_date && end_date.is_a?(String)
       queries = build_queries(query, start_date, end_date)
-      threads = queries.size if threads > queries.size
-      proxies = proxy ? Proxy::Pool.new : []
+      type = Type.new(type)
+      if threads > queries.size
+        logger.warn 'The maximum number of :threads is the number of dates between :start_date and :end_date.'
+        threads = queries.size
+      end
+      if proxy_enabled?
+        proxies = Proxy::Pool.new
+        logger.debug "Fetch #{proxies.size} proxies"
+      else
+        proxies = []
+        logger.debug 'Proxy disabled'
+      end
+      logger.debug "Cache #{cache_enabled? ? 'enabled' : 'disabled'}"
+
 
-      validate_options!(queries[0], start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads, proxy: proxy)
+      validate_options!(queries, type: type, start_date: start_date, end_date: end_date, lang: lang, limit: limit, threads: threads)
 
-      logger.debug "Fetch #{proxies.size} proxies" if proxy
       logger.info "The number of threads #{threads}"
 
       headers = {'User-Agent': USER_AGENT_LIST.sample, 'X-Requested-With': 'XMLHttpRequest'}
@@ -218,17 +231,25 @@ module Twitterscraper
         logger.debug "Set 'Thread.abort_on_exception' to true"
 
         Parallel.each(queries, in_threads: threads) do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           raise Parallel::Break if stop_requested?
         end
       else
         queries.each do |query|
-          main_loop(query, lang, limit, daily_limit, headers, proxies)
+          main_loop(query, lang, type, limit, daily_limit, headers, proxies)
           break if stop_requested?
         end
       end
 
-      @all_tweets.sort_by { |tweet| -tweet.created_at.to_i }
+      @all_tweets.sort_by { |tweet| (order == 'desc' ? -1 : 1) * tweet.created_at.to_i }
+    end
+
+    def search(query, start_date: nil, end_date: nil, lang: '', limit: 100, daily_limit: nil, order: 'desc', threads: 2)
+      query_tweets(query, type: 'search', start_date: start_date, end_date: end_date, lang: lang, limit: limit, daily_limit: daily_limit, order: order, threads: threads)
+    end
+
+    def user_timeline(screen_name, limit: 100, order: 'desc')
+      query_tweets(screen_name, type: 'user', start_date: nil, end_date: nil, lang: nil, limit: limit, daily_limit: nil, order: order, threads: 1)
     end
   end
 end

data/lib/twitterscraper/type.rb

@@ -0,0 +1,15 @@
+module Twitterscraper
+  class Type
+    def initialize(value)
+      @value = value
+    end
+
+    def search?
+      @value == 'search'
+    end
+
+    def user?
+      @value == 'user'
+    end
+  end
+end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Twitterscraper
-  VERSION = '0.12.0'
+  VERSION = '0.15.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitterscraper-ruby
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.15.2
 platform: ruby
 authors:
 - ts-3156
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -46,8 +46,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".irbrc"
+- ".rspec"
 - ".ruby-version"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
@@ -71,6 +73,7 @@ files:
 - lib/twitterscraper/query.rb
 - lib/twitterscraper/template.rb
 - lib/twitterscraper/tweet.rb
+- lib/twitterscraper/type.rb
 - lib/version.rb
 - twitterscraper-ruby.gemspec
 homepage: https://github.com/ts-3156/twitterscraper-ruby