x-twitter-scraper 0.10.5 → 0.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/README.md +49 -17
- data/lib/x_twitter_scraper/client.rb +0 -13
- data/lib/x_twitter_scraper/file_part.rb +0 -3
- data/lib/x_twitter_scraper/internal/transport/base_client.rb +3 -211
- data/lib/x_twitter_scraper/internal/transport/pooled_net_requester.rb +2 -56
- data/lib/x_twitter_scraper/internal/type/array_of.rb +1 -77
- data/lib/x_twitter_scraper/internal/type/base_model.rb +3 -133
- data/lib/x_twitter_scraper/internal/type/base_page.rb +0 -13
- data/lib/x_twitter_scraper/internal/type/boolean.rb +0 -38
- data/lib/x_twitter_scraper/internal/type/converter.rb +0 -136
- data/lib/x_twitter_scraper/internal/type/enum.rb +1 -80
- data/lib/x_twitter_scraper/internal/type/file_input.rb +0 -39
- data/lib/x_twitter_scraper/internal/type/hash_of.rb +1 -77
- data/lib/x_twitter_scraper/internal/type/request_parameters.rb +0 -13
- data/lib/x_twitter_scraper/internal/type/union.rb +0 -101
- data/lib/x_twitter_scraper/internal/type/unknown.rb +0 -40
- data/lib/x_twitter_scraper/internal/util.rb +2 -307
- data/lib/x_twitter_scraper/models/account_retrieve_response.rb +0 -3
- data/lib/x_twitter_scraper/models/compose_create_params.rb +0 -4
- data/lib/x_twitter_scraper/models/compose_create_response.rb +0 -4
- data/lib/x_twitter_scraper/models/credit_retrieve_balance_response.rb +0 -3
- data/lib/x_twitter_scraper/models/credit_topup_balance_response.rb +0 -3
- data/lib/x_twitter_scraper/models/draft_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/draw_export_params.rb +0 -3
- data/lib/x_twitter_scraper/models/draw_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/draw_run_response.rb +0 -3
- data/lib/x_twitter_scraper/models/embedded_tweet.rb +0 -3
- data/lib/x_twitter_scraper/models/event.rb +0 -3
- data/lib/x_twitter_scraper/models/event_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/extraction_estimate_cost_params.rb +0 -3
- data/lib/x_twitter_scraper/models/extraction_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/extraction_run_params.rb +0 -3
- data/lib/x_twitter_scraper/models/guest_wallet_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/guest_wallet_retrieve_status_response.rb +0 -3
- data/lib/x_twitter_scraper/models/guest_wallet_topup_response.rb +0 -3
- data/lib/x_twitter_scraper/models/radar_item.rb +0 -3
- data/lib/x_twitter_scraper/models/radar_retrieve_trending_topics_params.rb +0 -3
- data/lib/x_twitter_scraper/models/search_tweet.rb +0 -3
- data/lib/x_twitter_scraper/models/trend_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/webhook.rb +0 -3
- data/lib/x_twitter_scraper/models/x/communities/join_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/communities/join_delete_all_response.rb +0 -4
- data/lib/x_twitter_scraper/models/x/communities/tweet_list_by_community_params.rb +0 -4
- data/lib/x_twitter_scraper/models/x/communities/tweet_list_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/community_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/community_delete_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/community_retrieve_members_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/community_retrieve_moderators_params.rb +0 -4
- data/lib/x_twitter_scraper/models/x/community_retrieve_search_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/dm_retrieve_history_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/dm_retrieve_history_response.rb +0 -4
- data/lib/x_twitter_scraper/models/x/dm_send_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/list_retrieve_followers_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/list_retrieve_members_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/list_retrieve_tweets_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/media_upload_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/profile_update_avatar_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/profile_update_banner_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/profile_update_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_create_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_delete_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_detail.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_get_favoriters_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_get_quotes_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_get_replies_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_get_retweeters_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_get_thread_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_retrieve_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweet_search_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweets/like_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweets/like_delete_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweets/retweet_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/tweets/retweet_delete_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_remove_follower_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_batch_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_batch_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_followers_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_followers_you_know_params.rb +0 -4
- data/lib/x_twitter_scraper/models/x/user_retrieve_following_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_likes_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_media_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_mentions_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_replies_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_search_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_tweets_params.rb +0 -3
- data/lib/x_twitter_scraper/models/x/user_retrieve_verified_followers_params.rb +0 -4
- data/lib/x_twitter_scraper/models/x/users/follow_create_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/users/follow_delete_all_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/write_action_retrieve_response.rb +0 -3
- data/lib/x_twitter_scraper/models/x/x_account.rb +0 -3
- data/lib/x_twitter_scraper/models/x_get_article_response.rb +0 -4
- data/lib/x_twitter_scraper/request_options.rb +0 -2
- data/lib/x_twitter_scraper/resources/drafts.rb +0 -3
- data/lib/x_twitter_scraper/resources/draws.rb +0 -6
- data/lib/x_twitter_scraper/resources/events.rb +0 -3
- data/lib/x_twitter_scraper/resources/extractions.rb +0 -9
- data/lib/x_twitter_scraper/resources/guest_wallets.rb +0 -6
- data/lib/x_twitter_scraper/resources/radar.rb +0 -3
- data/lib/x_twitter_scraper/resources/support/tickets.rb +0 -6
- data/lib/x_twitter_scraper/resources/trends.rb +0 -3
- data/lib/x_twitter_scraper/resources/x/communities/join.rb +0 -6
- data/lib/x_twitter_scraper/resources/x/communities/tweets.rb +0 -7
- data/lib/x_twitter_scraper/resources/x/communities.rb +0 -16
- data/lib/x_twitter_scraper/resources/x/dm.rb +0 -6
- data/lib/x_twitter_scraper/resources/x/lists.rb +0 -9
- data/lib/x_twitter_scraper/resources/x/media.rb +0 -3
- data/lib/x_twitter_scraper/resources/x/profile.rb +0 -9
- data/lib/x_twitter_scraper/resources/x/tweets/like.rb +0 -6
- data/lib/x_twitter_scraper/resources/x/tweets/retweet.rb +0 -6
- data/lib/x_twitter_scraper/resources/x/tweets.rb +0 -24
- data/lib/x_twitter_scraper/resources/x/users/follow.rb +0 -6
- data/lib/x_twitter_scraper/resources/x/users.rb +0 -38
- data/lib/x_twitter_scraper/resources/x.rb +0 -3
- data/lib/x_twitter_scraper/version.rb +1 -1
- data/lib/x_twitter_scraper.rb +0 -5
- data/rbi/x_twitter_scraper/client.rbi +0 -4
- data/rbi/x_twitter_scraper/file_part.rbi +0 -1
- data/rbi/x_twitter_scraper/internal/transport/base_client.rbi +0 -19
- data/rbi/x_twitter_scraper/internal/transport/pooled_net_requester.rbi +0 -9
- data/rbi/x_twitter_scraper/internal/type/array_of.rbi +0 -10
- data/rbi/x_twitter_scraper/internal/type/base_model.rbi +0 -47
- data/rbi/x_twitter_scraper/internal/type/base_page.rbi +0 -4
- data/rbi/x_twitter_scraper/internal/type/boolean.rbi +0 -8
- data/rbi/x_twitter_scraper/internal/type/converter.rbi +0 -36
- data/rbi/x_twitter_scraper/internal/type/enum.rbi +0 -20
- data/rbi/x_twitter_scraper/internal/type/file_input.rbi +0 -10
- data/rbi/x_twitter_scraper/internal/type/hash_of.rbi +0 -10
- data/rbi/x_twitter_scraper/internal/type/request_parameters.rbi +0 -4
- data/rbi/x_twitter_scraper/internal/type/union.rbi +0 -18
- data/rbi/x_twitter_scraper/internal/type/unknown.rbi +0 -8
- data/rbi/x_twitter_scraper/internal/util.rbi +0 -75
- data/rbi/x_twitter_scraper/internal.rbi +0 -2
- data/rbi/x_twitter_scraper/request_options.rbi +0 -1
- data/sig/x_twitter_scraper/internal/transport/base_client.rbs +0 -1
- metadata +2 -2
|
@@ -6,18 +6,9 @@
|
|
|
6
6
|
|
|
7
7
|
module XTwitterScraper
|
|
8
8
|
module Internal
|
|
9
|
-
# @api private
|
|
10
9
|
module Util
|
|
11
|
-
# @api private
|
|
12
|
-
#
|
|
13
|
-
# @return [Float]
|
|
14
10
|
def self.monotonic_secs = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
15
11
|
|
|
16
|
-
# @api private
|
|
17
|
-
#
|
|
18
|
-
# @param ns [Module, Class]
|
|
19
|
-
#
|
|
20
|
-
# @return [Enumerable<Module, Class>]
|
|
21
12
|
def self.walk_namespaces(ns)
|
|
22
13
|
ns.constants(false).lazy.flat_map do
|
|
23
14
|
case (c = ns.const_get(_1, false))
|
|
@@ -31,9 +22,6 @@ module XTwitterScraper
|
|
|
31
22
|
end
|
|
32
23
|
|
|
33
24
|
class << self
|
|
34
|
-
# @api private
|
|
35
|
-
#
|
|
36
|
-
# @return [String]
|
|
37
25
|
def arch
|
|
38
26
|
case (arch = RbConfig::CONFIG["arch"])&.downcase
|
|
39
27
|
in nil
|
|
@@ -49,9 +37,6 @@ module XTwitterScraper
|
|
|
49
37
|
end
|
|
50
38
|
end
|
|
51
39
|
|
|
52
|
-
# @api private
|
|
53
|
-
#
|
|
54
|
-
# @return [String]
|
|
55
40
|
def os
|
|
56
41
|
case (host = RbConfig::CONFIG["host_os"])&.downcase
|
|
57
42
|
in nil
|
|
@@ -73,11 +58,6 @@ module XTwitterScraper
|
|
|
73
58
|
end
|
|
74
59
|
|
|
75
60
|
class << self
|
|
76
|
-
# @api private
|
|
77
|
-
#
|
|
78
|
-
# @param input [Object]
|
|
79
|
-
#
|
|
80
|
-
# @return [Boolean]
|
|
81
61
|
def primitive?(input)
|
|
82
62
|
case input
|
|
83
63
|
in true | false | Numeric | Symbol | String
|
|
@@ -87,11 +67,6 @@ module XTwitterScraper
|
|
|
87
67
|
end
|
|
88
68
|
end
|
|
89
69
|
|
|
90
|
-
# @api private
|
|
91
|
-
#
|
|
92
|
-
# @param input [String, Boolean]
|
|
93
|
-
#
|
|
94
|
-
# @return [Boolean, Object]
|
|
95
70
|
def coerce_boolean(input)
|
|
96
71
|
case input.is_a?(String) ? input.downcase : input
|
|
97
72
|
in "true"
|
|
@@ -103,12 +78,6 @@ module XTwitterScraper
|
|
|
103
78
|
end
|
|
104
79
|
end
|
|
105
80
|
|
|
106
|
-
# @api private
|
|
107
|
-
#
|
|
108
|
-
# @param input [String, Boolean]
|
|
109
|
-
#
|
|
110
|
-
# @raise [ArgumentError]
|
|
111
|
-
# @return [Boolean, nil]
|
|
112
81
|
def coerce_boolean!(input)
|
|
113
82
|
case coerce_boolean(input)
|
|
114
83
|
in true | false | nil => coerced
|
|
@@ -118,29 +87,14 @@ module XTwitterScraper
|
|
|
118
87
|
end
|
|
119
88
|
end
|
|
120
89
|
|
|
121
|
-
# @api private
|
|
122
|
-
#
|
|
123
|
-
# @param input [String, Integer]
|
|
124
|
-
#
|
|
125
|
-
# @return [Integer, Object]
|
|
126
90
|
def coerce_integer(input)
|
|
127
91
|
Integer(input, exception: false) || input
|
|
128
92
|
end
|
|
129
93
|
|
|
130
|
-
# @api private
|
|
131
|
-
#
|
|
132
|
-
# @param input [String, Integer, Float]
|
|
133
|
-
#
|
|
134
|
-
# @return [Float, Object]
|
|
135
94
|
def coerce_float(input)
|
|
136
95
|
Float(input, exception: false) || input
|
|
137
96
|
end
|
|
138
97
|
|
|
139
|
-
# @api private
|
|
140
|
-
#
|
|
141
|
-
# @param input [Object]
|
|
142
|
-
#
|
|
143
|
-
# @return [Hash{Object=>Object}, Object]
|
|
144
98
|
def coerce_hash(input)
|
|
145
99
|
case input
|
|
146
100
|
in NilClass | Array | Set | Enumerator | StringIO | IO
|
|
@@ -150,12 +104,6 @@ module XTwitterScraper
|
|
|
150
104
|
end
|
|
151
105
|
end
|
|
152
106
|
|
|
153
|
-
# @api private
|
|
154
|
-
#
|
|
155
|
-
# @param input [Object]
|
|
156
|
-
#
|
|
157
|
-
# @raise [ArgumentError]
|
|
158
|
-
# @return [Hash{Object=>Object}, nil]
|
|
159
107
|
def coerce_hash!(input)
|
|
160
108
|
case coerce_hash(input)
|
|
161
109
|
in Hash | nil => coerced
|
|
@@ -168,13 +116,6 @@ module XTwitterScraper
|
|
|
168
116
|
end
|
|
169
117
|
|
|
170
118
|
class << self
|
|
171
|
-
# @api private
|
|
172
|
-
#
|
|
173
|
-
# @param lhs [Object]
|
|
174
|
-
# @param rhs [Object]
|
|
175
|
-
# @param concat [Boolean]
|
|
176
|
-
#
|
|
177
|
-
# @return [Object]
|
|
178
119
|
private def deep_merge_lr(lhs, rhs, concat: false)
|
|
179
120
|
case [lhs, rhs, concat]
|
|
180
121
|
in [Hash, Hash, _]
|
|
@@ -186,18 +127,6 @@ module XTwitterScraper
|
|
|
186
127
|
end
|
|
187
128
|
end
|
|
188
129
|
|
|
189
|
-
# @api private
|
|
190
|
-
#
|
|
191
|
-
# Recursively merge one hash with another. If the values at a given key are not
|
|
192
|
-
# both hashes, just take the new value.
|
|
193
|
-
#
|
|
194
|
-
# @param values [Array<Object>]
|
|
195
|
-
#
|
|
196
|
-
# @param sentinel [Object, nil] the value to return if no values are provided.
|
|
197
|
-
#
|
|
198
|
-
# @param concat [Boolean] whether to merge sequences by concatenation.
|
|
199
|
-
#
|
|
200
|
-
# @return [Object]
|
|
201
130
|
def deep_merge(*values, sentinel: nil, concat: false)
|
|
202
131
|
case values
|
|
203
132
|
in [value, *values]
|
|
@@ -209,13 +138,6 @@ module XTwitterScraper
|
|
|
209
138
|
end
|
|
210
139
|
end
|
|
211
140
|
|
|
212
|
-
# @api private
|
|
213
|
-
#
|
|
214
|
-
# @param data [Hash{Symbol=>Object}, Array<Object>, Object]
|
|
215
|
-
# @param pick [Symbol, Integer, Array<Symbol, Integer>, Proc, nil]
|
|
216
|
-
# @param blk [Proc, nil]
|
|
217
|
-
#
|
|
218
|
-
# @return [Object, nil]
|
|
219
141
|
def dig(data, pick, &blk)
|
|
220
142
|
case [data, pick]
|
|
221
143
|
in [_, nil]
|
|
@@ -241,35 +163,17 @@ module XTwitterScraper
|
|
|
241
163
|
end
|
|
242
164
|
end
|
|
243
165
|
|
|
244
|
-
# @type [Regexp]
|
|
245
|
-
#
|
|
246
|
-
# https://www.rfc-editor.org/rfc/rfc3986.html#section-3.3
|
|
247
166
|
RFC_3986_NOT_PCHARS = /[^A-Za-z0-9\-._~!$&'()*+,;=:@]+/
|
|
248
167
|
|
|
249
168
|
class << self
|
|
250
|
-
# @api private
|
|
251
|
-
#
|
|
252
|
-
# @param uri [URI::Generic]
|
|
253
|
-
#
|
|
254
|
-
# @return [String]
|
|
255
169
|
def uri_origin(uri)
|
|
256
170
|
"#{uri.scheme}://#{uri.host}#{":#{uri.port}" unless uri.port == uri.default_port}"
|
|
257
171
|
end
|
|
258
172
|
|
|
259
|
-
# @api private
|
|
260
|
-
#
|
|
261
|
-
# @param path [String, Integer]
|
|
262
|
-
#
|
|
263
|
-
# @return [String]
|
|
264
173
|
def encode_path(path)
|
|
265
174
|
path.to_s.gsub(XTwitterScraper::Internal::Util::RFC_3986_NOT_PCHARS) { ERB::Util.url_encode(_1) }
|
|
266
175
|
end
|
|
267
176
|
|
|
268
|
-
# @api private
|
|
269
|
-
#
|
|
270
|
-
# @param path [String, Array<String>]
|
|
271
|
-
#
|
|
272
|
-
# @return [String]
|
|
273
177
|
def interpolate_path(path)
|
|
274
178
|
case path
|
|
275
179
|
in String
|
|
@@ -284,82 +188,25 @@ module XTwitterScraper
|
|
|
284
188
|
end
|
|
285
189
|
|
|
286
190
|
class << self
|
|
287
|
-
# @api private
|
|
288
|
-
#
|
|
289
|
-
# @param query [String, nil]
|
|
290
|
-
#
|
|
291
|
-
# @return [Hash{String=>Array<String>}]
|
|
292
191
|
def decode_query(query)
|
|
293
192
|
CGI.parse(query.to_s)
|
|
294
193
|
end
|
|
295
194
|
|
|
296
|
-
# @api private
|
|
297
|
-
#
|
|
298
|
-
# @param query [Hash{String=>Array<String>, String, nil}, nil]
|
|
299
|
-
#
|
|
300
|
-
# @return [String, nil]
|
|
301
195
|
def encode_query(query)
|
|
302
196
|
query.to_h.empty? ? nil : URI.encode_www_form(query)
|
|
303
197
|
end
|
|
304
198
|
end
|
|
305
199
|
|
|
306
200
|
class << self
|
|
307
|
-
# @api private
|
|
308
|
-
#
|
|
309
|
-
# @param url [URI::Generic, String]
|
|
310
|
-
#
|
|
311
|
-
# @return [Hash{Symbol=>String, Integer, nil}]
|
|
312
201
|
def parse_uri(url)
|
|
313
202
|
parsed = URI::Generic.component.zip(URI.split(url)).to_h
|
|
314
203
|
{**parsed, query: decode_query(parsed.fetch(:query))}
|
|
315
204
|
end
|
|
316
205
|
|
|
317
|
-
# @api private
|
|
318
|
-
#
|
|
319
|
-
# @param parsed [Hash{Symbol=>String, Integer, nil}] .
|
|
320
|
-
#
|
|
321
|
-
# @option parsed [String, nil] :scheme
|
|
322
|
-
#
|
|
323
|
-
# @option parsed [String, nil] :host
|
|
324
|
-
#
|
|
325
|
-
# @option parsed [Integer, nil] :port
|
|
326
|
-
#
|
|
327
|
-
# @option parsed [String, nil] :path
|
|
328
|
-
#
|
|
329
|
-
# @option parsed [Hash{String=>Array<String>}] :query
|
|
330
|
-
#
|
|
331
|
-
# @return [URI::Generic]
|
|
332
206
|
def unparse_uri(parsed)
|
|
333
207
|
URI::Generic.build(**parsed, query: encode_query(parsed.fetch(:query)))
|
|
334
208
|
end
|
|
335
209
|
|
|
336
|
-
# @api private
|
|
337
|
-
#
|
|
338
|
-
# @param lhs [Hash{Symbol=>String, Integer, nil}] .
|
|
339
|
-
#
|
|
340
|
-
# @option lhs [String, nil] :scheme
|
|
341
|
-
#
|
|
342
|
-
# @option lhs [String, nil] :host
|
|
343
|
-
#
|
|
344
|
-
# @option lhs [Integer, nil] :port
|
|
345
|
-
#
|
|
346
|
-
# @option lhs [String, nil] :path
|
|
347
|
-
#
|
|
348
|
-
# @option lhs [Hash{String=>Array<String>}] :query
|
|
349
|
-
#
|
|
350
|
-
# @param rhs [Hash{Symbol=>String, Integer, nil}] .
|
|
351
|
-
#
|
|
352
|
-
# @option rhs [String, nil] :scheme
|
|
353
|
-
#
|
|
354
|
-
# @option rhs [String, nil] :host
|
|
355
|
-
#
|
|
356
|
-
# @option rhs [Integer, nil] :port
|
|
357
|
-
#
|
|
358
|
-
# @option rhs [String, nil] :path
|
|
359
|
-
#
|
|
360
|
-
# @option rhs [Hash{String=>Array<String>}] :query
|
|
361
|
-
#
|
|
362
|
-
# @return [URI::Generic]
|
|
363
210
|
def join_parsed_uri(lhs, rhs)
|
|
364
211
|
base_path, base_query = lhs.fetch_values(:path, :query)
|
|
365
212
|
slashed = base_path.end_with?("/") ? base_path : "#{base_path}/"
|
|
@@ -382,11 +229,6 @@ module XTwitterScraper
|
|
|
382
229
|
end
|
|
383
230
|
|
|
384
231
|
class << self
|
|
385
|
-
# @api private
|
|
386
|
-
#
|
|
387
|
-
# @param headers [Hash{String=>String, Integer, Array<String, Integer, nil>, nil}]
|
|
388
|
-
#
|
|
389
|
-
# @return [Hash{String=>String}]
|
|
390
232
|
def normalized_headers(*headers)
|
|
391
233
|
{}.merge(*headers.compact).to_h do |key, val|
|
|
392
234
|
value =
|
|
@@ -401,16 +243,9 @@ module XTwitterScraper
|
|
|
401
243
|
end
|
|
402
244
|
end
|
|
403
245
|
|
|
404
|
-
# @api private
|
|
405
|
-
#
|
|
406
|
-
# An adapter that satisfies the IO interface required by `::IO.copy_stream`
|
|
407
246
|
class ReadIOAdapter
|
|
408
|
-
# @api private
|
|
409
|
-
#
|
|
410
|
-
# @return [Boolean, nil]
|
|
411
247
|
def close? = @closing
|
|
412
248
|
|
|
413
|
-
# @api private
|
|
414
249
|
def close
|
|
415
250
|
case @stream
|
|
416
251
|
in Enumerator
|
|
@@ -421,11 +256,6 @@ module XTwitterScraper
|
|
|
421
256
|
end
|
|
422
257
|
end
|
|
423
258
|
|
|
424
|
-
# @api private
|
|
425
|
-
#
|
|
426
|
-
# @param max_len [Integer, nil]
|
|
427
|
-
#
|
|
428
|
-
# @return [String]
|
|
429
259
|
private def read_enum(max_len)
|
|
430
260
|
case max_len
|
|
431
261
|
in nil
|
|
@@ -439,12 +269,6 @@ module XTwitterScraper
|
|
|
439
269
|
@buf.slice!(0..)
|
|
440
270
|
end
|
|
441
271
|
|
|
442
|
-
# @api private
|
|
443
|
-
#
|
|
444
|
-
# @param max_len [Integer, nil]
|
|
445
|
-
# @param out_string [String, nil]
|
|
446
|
-
#
|
|
447
|
-
# @return [String, nil]
|
|
448
272
|
def read(max_len = nil, out_string = nil)
|
|
449
273
|
case @stream
|
|
450
274
|
in nil
|
|
@@ -463,12 +287,6 @@ module XTwitterScraper
|
|
|
463
287
|
.tap(&@blk)
|
|
464
288
|
end
|
|
465
289
|
|
|
466
|
-
# @api private
|
|
467
|
-
#
|
|
468
|
-
# @param src [String, Pathname, StringIO, Enumerable<String>]
|
|
469
|
-
# @param blk [Proc]
|
|
470
|
-
#
|
|
471
|
-
# @yieldparam [String]
|
|
472
290
|
def initialize(src, &blk)
|
|
473
291
|
@stream =
|
|
474
292
|
case src
|
|
@@ -486,10 +304,6 @@ module XTwitterScraper
|
|
|
486
304
|
end
|
|
487
305
|
|
|
488
306
|
class << self
|
|
489
|
-
# @param blk [Proc]
|
|
490
|
-
#
|
|
491
|
-
# @yieldparam [Enumerator::Yielder]
|
|
492
|
-
# @return [Enumerable<String>]
|
|
493
307
|
def writable_enum(&blk)
|
|
494
308
|
Enumerator.new do |y|
|
|
495
309
|
y.define_singleton_method(:write) do
|
|
@@ -502,30 +316,16 @@ module XTwitterScraper
|
|
|
502
316
|
end
|
|
503
317
|
end
|
|
504
318
|
|
|
505
|
-
# @type [Regexp]
|
|
506
319
|
JSON_CONTENT = %r{^application/(?:[a-zA-Z0-9.-]+\+)?json(?!l)}
|
|
507
|
-
# @type [Regexp]
|
|
508
320
|
JSONL_CONTENT = %r{\Aapplication/(?:x-(?:n|l)djson|(?:x-)?jsonl)}
|
|
509
321
|
|
|
510
322
|
class << self
|
|
511
|
-
# @api private
|
|
512
|
-
#
|
|
513
|
-
# @param query [Hash{Symbol=>Object}]
|
|
514
|
-
#
|
|
515
|
-
# @return [Hash{Symbol=>Object}]
|
|
516
323
|
def encode_query_params(query)
|
|
517
324
|
out = {}
|
|
518
325
|
query.each { write_query_param_element!(out, _1, _2) }
|
|
519
326
|
out
|
|
520
327
|
end
|
|
521
328
|
|
|
522
|
-
# @api private
|
|
523
|
-
#
|
|
524
|
-
# @param collection [Hash{Symbol=>Object}]
|
|
525
|
-
# @param key [String]
|
|
526
|
-
# @param element [Object]
|
|
527
|
-
#
|
|
528
|
-
# @return [nil]
|
|
529
329
|
private def write_query_param_element!(collection, key, element)
|
|
530
330
|
case element
|
|
531
331
|
in Hash
|
|
@@ -539,12 +339,6 @@ module XTwitterScraper
|
|
|
539
339
|
end
|
|
540
340
|
end
|
|
541
341
|
|
|
542
|
-
# @api private
|
|
543
|
-
#
|
|
544
|
-
# @param y [Enumerator::Yielder]
|
|
545
|
-
# @param val [Object]
|
|
546
|
-
# @param closing [Array<Proc>]
|
|
547
|
-
# @param content_type [String, nil]
|
|
548
342
|
private def write_multipart_content(y, val:, closing:, content_type: nil)
|
|
549
343
|
content_line = "Content-Type: %s\r\n\r\n"
|
|
550
344
|
|
|
@@ -577,13 +371,6 @@ module XTwitterScraper
|
|
|
577
371
|
y << "\r\n"
|
|
578
372
|
end
|
|
579
373
|
|
|
580
|
-
# @api private
|
|
581
|
-
#
|
|
582
|
-
# @param y [Enumerator::Yielder]
|
|
583
|
-
# @param boundary [String]
|
|
584
|
-
# @param key [Symbol, String]
|
|
585
|
-
# @param val [Object]
|
|
586
|
-
# @param closing [Array<Proc>]
|
|
587
374
|
private def write_multipart_chunk(y, boundary:, key:, val:, closing:)
|
|
588
375
|
y << "--#{boundary}\r\n"
|
|
589
376
|
y << "Content-Disposition: form-data"
|
|
@@ -606,16 +393,9 @@ module XTwitterScraper
|
|
|
606
393
|
write_multipart_content(y, val: val, closing: closing)
|
|
607
394
|
end
|
|
608
395
|
|
|
609
|
-
# @api private
|
|
610
|
-
#
|
|
611
396
|
# https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.1.md#special-considerations-for-multipart-content
|
|
612
|
-
#
|
|
613
|
-
# @param body [Object]
|
|
614
|
-
#
|
|
615
|
-
# @return [Array(String, Enumerable<String>)]
|
|
616
397
|
private def encode_multipart_streaming(body)
|
|
617
398
|
# rubocop:disable Style/CaseEquality
|
|
618
|
-
# RFC 1521 Section 7.2.1 says we should have 70 char maximum for boundary length
|
|
619
399
|
boundary = SecureRandom.urlsafe_base64(46)
|
|
620
400
|
|
|
621
401
|
closing = []
|
|
@@ -643,12 +423,6 @@ module XTwitterScraper
|
|
|
643
423
|
# rubocop:enable Style/CaseEquality
|
|
644
424
|
end
|
|
645
425
|
|
|
646
|
-
# @api private
|
|
647
|
-
#
|
|
648
|
-
# @param headers [Hash{String=>String}]
|
|
649
|
-
# @param body [Object]
|
|
650
|
-
#
|
|
651
|
-
# @return [Object]
|
|
652
426
|
def encode_content(headers, body)
|
|
653
427
|
# rubocop:disable Style/CaseEquality
|
|
654
428
|
# rubocop:disable Layout/LineLength
|
|
@@ -675,12 +449,6 @@ module XTwitterScraper
|
|
|
675
449
|
# rubocop:enable Style/CaseEquality
|
|
676
450
|
end
|
|
677
451
|
|
|
678
|
-
# @api private
|
|
679
|
-
#
|
|
680
|
-
# https://www.iana.org/assignments/character-sets/character-sets.xhtml
|
|
681
|
-
#
|
|
682
|
-
# @param content_type [String]
|
|
683
|
-
# @param text [String]
|
|
684
452
|
def force_charset!(content_type, text:)
|
|
685
453
|
charset = /charset=([^;\s]+)/.match(content_type)&.captures&.first
|
|
686
454
|
|
|
@@ -694,16 +462,6 @@ module XTwitterScraper
|
|
|
694
462
|
end
|
|
695
463
|
end
|
|
696
464
|
|
|
697
|
-
# @api private
|
|
698
|
-
#
|
|
699
|
-
# Assumes each chunk in stream has `Encoding::BINARY`.
|
|
700
|
-
#
|
|
701
|
-
# @param headers [Hash{String=>String}]
|
|
702
|
-
# @param stream [Enumerable<String>]
|
|
703
|
-
# @param suppress_error [Boolean]
|
|
704
|
-
#
|
|
705
|
-
# @raise [JSON::ParserError]
|
|
706
|
-
# @return [Object]
|
|
707
465
|
def decode_content(headers, stream:, suppress_error: false)
|
|
708
466
|
case (content_type = headers["content-type"])
|
|
709
467
|
in XTwitterScraper::Internal::Util::JSON_CONTENT
|
|
@@ -736,15 +494,6 @@ module XTwitterScraper
|
|
|
736
494
|
end
|
|
737
495
|
|
|
738
496
|
class << self
|
|
739
|
-
# @api private
|
|
740
|
-
#
|
|
741
|
-
# https://doc.rust-lang.org/std/iter/trait.FusedIterator.html
|
|
742
|
-
#
|
|
743
|
-
# @param enum [Enumerable<Object>]
|
|
744
|
-
# @param external [Boolean]
|
|
745
|
-
# @param close [Proc]
|
|
746
|
-
#
|
|
747
|
-
# @return [Enumerable<Object>]
|
|
748
497
|
def fused_enum(enum, external: false, &close)
|
|
749
498
|
fused = false
|
|
750
499
|
iter = Enumerator.new do |y|
|
|
@@ -768,24 +517,13 @@ module XTwitterScraper
|
|
|
768
517
|
iter
|
|
769
518
|
end
|
|
770
519
|
|
|
771
|
-
# @api private
|
|
772
|
-
#
|
|
773
|
-
# @param enum [Enumerable<Object>, nil]
|
|
774
520
|
def close_fused!(enum)
|
|
775
521
|
return unless enum.is_a?(Enumerator)
|
|
776
522
|
|
|
777
|
-
# rubocop:disable Lint/UnreachableLoop
|
|
523
|
+
# rubocop:disable-next Lint/UnreachableLoop
|
|
778
524
|
enum.rewind.each { break }
|
|
779
|
-
# rubocop:enable Lint/UnreachableLoop
|
|
780
525
|
end
|
|
781
526
|
|
|
782
|
-
# @api private
|
|
783
|
-
#
|
|
784
|
-
# @param enum [Enumerable<Object>, nil]
|
|
785
|
-
# @param blk [Proc]
|
|
786
|
-
#
|
|
787
|
-
# @yieldparam [Enumerator::Yielder]
|
|
788
|
-
# @return [Enumerable<Object>]
|
|
789
527
|
def chain_fused(enum, &blk)
|
|
790
528
|
iter = Enumerator.new { blk.call(_1) }
|
|
791
529
|
fused_enum(iter) { close_fused!(enum) }
|
|
@@ -793,16 +531,6 @@ module XTwitterScraper
|
|
|
793
531
|
end
|
|
794
532
|
|
|
795
533
|
class << self
|
|
796
|
-
# @api private
|
|
797
|
-
#
|
|
798
|
-
# Assumes Strings have been forced into having `Encoding::BINARY`.
|
|
799
|
-
#
|
|
800
|
-
# This decoder is responsible for reassembling lines split across multiple
|
|
801
|
-
# fragments.
|
|
802
|
-
#
|
|
803
|
-
# @param enum [Enumerable<String>]
|
|
804
|
-
#
|
|
805
|
-
# @return [Enumerable<String>]
|
|
806
534
|
def decode_lines(enum)
|
|
807
535
|
re = /(\r\n|\r|\n)/
|
|
808
536
|
buffer = String.new
|
|
@@ -832,15 +560,6 @@ module XTwitterScraper
|
|
|
832
560
|
end
|
|
833
561
|
end
|
|
834
562
|
|
|
835
|
-
# @api private
|
|
836
|
-
#
|
|
837
|
-
# https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
|
|
838
|
-
#
|
|
839
|
-
# Assumes that `lines` has been decoded with `#decode_lines`.
|
|
840
|
-
#
|
|
841
|
-
# @param lines [Enumerable<String>]
|
|
842
|
-
#
|
|
843
|
-
# @return [Enumerable<Hash{Symbol=>Object}>]
|
|
844
563
|
def decode_sse(lines)
|
|
845
564
|
# rubocop:disable Metrics/BlockLength
|
|
846
565
|
chain_fused(lines) do |y|
|
|
@@ -878,19 +597,12 @@ module XTwitterScraper
|
|
|
878
597
|
end
|
|
879
598
|
end
|
|
880
599
|
|
|
881
|
-
# @api private
|
|
882
600
|
module SorbetRuntimeSupport
|
|
883
601
|
class MissingSorbetRuntimeError < ::RuntimeError
|
|
884
602
|
end
|
|
885
603
|
|
|
886
|
-
# @api private
|
|
887
|
-
#
|
|
888
|
-
# @return [Hash{Symbol=>Object}]
|
|
889
604
|
private def sorbet_runtime_constants = @sorbet_runtime_constants ||= {}
|
|
890
605
|
|
|
891
|
-
# @api private
|
|
892
|
-
#
|
|
893
|
-
# @param name [Symbol]
|
|
894
606
|
def const_missing(name)
|
|
895
607
|
super unless sorbet_runtime_constants.key?(name)
|
|
896
608
|
|
|
@@ -902,30 +614,13 @@ module XTwitterScraper
|
|
|
902
614
|
sorbet_runtime_constants.fetch(name).call
|
|
903
615
|
end
|
|
904
616
|
|
|
905
|
-
# @api private
|
|
906
|
-
#
|
|
907
|
-
# @param name [Symbol]
|
|
908
|
-
#
|
|
909
|
-
# @return [Boolean]
|
|
910
617
|
def sorbet_constant_defined?(name) = sorbet_runtime_constants.key?(name)
|
|
911
618
|
|
|
912
|
-
# @api private
|
|
913
|
-
#
|
|
914
|
-
# @param name [Symbol]
|
|
915
|
-
# @param blk [Proc]
|
|
916
619
|
def define_sorbet_constant!(name, &blk) = sorbet_runtime_constants.store(name, blk)
|
|
917
620
|
|
|
918
|
-
|
|
919
|
-
#
|
|
920
|
-
# @return [Object]
|
|
921
|
-
def to_sorbet_type = raise NotImplementedError
|
|
621
|
+
def to_sorbet_type = raise(NotImplementedError)
|
|
922
622
|
|
|
923
623
|
class << self
|
|
924
|
-
# @api private
|
|
925
|
-
#
|
|
926
|
-
# @param type [XTwitterScraper::Internal::Util::SorbetRuntimeSupport, Object]
|
|
927
|
-
#
|
|
928
|
-
# @return [Object]
|
|
929
624
|
def to_sorbet_type(type)
|
|
930
625
|
case type
|
|
931
626
|
in XTwitterScraper::Internal::Util::SorbetRuntimeSupport
|
|
@@ -48,9 +48,6 @@ module XTwitterScraper
|
|
|
48
48
|
optional :x_username, String, api_name: :xUsername
|
|
49
49
|
|
|
50
50
|
# @!method initialize(monitor_billing:, monitors_allowed:, monitors_used:, plan:, credit_info: nil, x_username: nil)
|
|
51
|
-
# Some parameter documentations has been truncated, see
|
|
52
|
-
# {XTwitterScraper::Models::AccountRetrieveResponse} for more details.
|
|
53
|
-
#
|
|
54
51
|
# @param monitor_billing [XTwitterScraper::Models::AccountRetrieveResponse::MonitorBilling]
|
|
55
52
|
#
|
|
56
53
|
# @param monitors_allowed [Integer] Deprecated. Monitor slots are unlimited, so this is always Number.MAX_SAFE_INTEG
|
|
@@ -194,10 +194,6 @@ module XTwitterScraper
|
|
|
194
194
|
optional :has_media, XTwitterScraper::Internal::Type::Boolean, api_name: :hasMedia
|
|
195
195
|
|
|
196
196
|
# @!method initialize(draft:, has_link: nil, has_media: nil, step: :score)
|
|
197
|
-
# Some parameter documentations has been truncated, see
|
|
198
|
-
# {XTwitterScraper::Models::ComposeCreateParams::Body::ComposeScoreRequest} for
|
|
199
|
-
# more details.
|
|
200
|
-
#
|
|
201
197
|
# @param draft [String] Full post text for deterministic editorial checks.
|
|
202
198
|
#
|
|
203
199
|
# @param has_link [Boolean] True when a separate link card is attached.
|
|
@@ -106,10 +106,6 @@ module XTwitterScraper
|
|
|
106
106
|
optional :style_tweets, XTwitterScraper::Internal::Type::ArrayOf[String], api_name: :styleTweets
|
|
107
107
|
|
|
108
108
|
# @!method initialize(content_rules:, engagement_multipliers:, engagement_velocity:, follow_up_questions:, intent_url:, next_step:, radar_recommendations:, scorer_weights:, source:, top_penalties:, saved_styles: nil, style_note: nil, style_tweets: nil)
|
|
109
|
-
# Some parameter documentations has been truncated, see
|
|
110
|
-
# {XTwitterScraper::Models::ComposeCreateResponse::ComposePrepareResult} for more
|
|
111
|
-
# details.
|
|
112
|
-
#
|
|
113
109
|
# @param content_rules [Array<XTwitterScraper::Models::ComposeCreateResponse::ComposePrepareResult::ContentRule>] Xquik editorial heuristics, ordered for the goal.
|
|
114
110
|
#
|
|
115
111
|
# @param engagement_multipliers [Array<XTwitterScraper::Models::ComposeCreateResponse::ComposePrepareResult::EngagementMultiplier>] Published engagement signal names. Production multipliers are not published.
|
|
@@ -46,9 +46,6 @@ module XTwitterScraper
|
|
|
46
46
|
required :lifetime_used, String
|
|
47
47
|
|
|
48
48
|
# @!method initialize(auto_topup_amount_dollars:, auto_topup_enabled:, auto_topup_threshold:, balance:, lifetime_purchased:, lifetime_used:)
|
|
49
|
-
# Some parameter documentations has been truncated, see
|
|
50
|
-
# {XTwitterScraper::Models::CreditRetrieveBalanceResponse} for more details.
|
|
51
|
-
#
|
|
52
49
|
# @param auto_topup_amount_dollars [Float] Configured dollar amount for each automatic top-up.
|
|
53
50
|
#
|
|
54
51
|
# @param auto_topup_enabled [Boolean]
|
|
@@ -22,9 +22,6 @@ module XTwitterScraper
|
|
|
22
22
|
required :url, String
|
|
23
23
|
|
|
24
24
|
# @!method initialize(redirect_url:, url:)
|
|
25
|
-
# Some parameter documentations has been truncated, see
|
|
26
|
-
# {XTwitterScraper::Models::CreditTopupBalanceResponse} for more details.
|
|
27
|
-
#
|
|
28
25
|
# @param redirect_url [String] Stable Xquik redirect URL for the active checkout.
|
|
29
26
|
#
|
|
30
27
|
# @param url [String] Same stable Xquik redirect URL as redirect_url. The response never exposes the h
|
|
@@ -27,9 +27,6 @@ module XTwitterScraper
|
|
|
27
27
|
optional :limit, Integer
|
|
28
28
|
|
|
29
29
|
# @!method initialize(after_cursor: nil, limit: nil, request_options: {})
|
|
30
|
-
# Some parameter documentations has been truncated, see
|
|
31
|
-
# {XTwitterScraper::Models::DraftListParams} for more details.
|
|
32
|
-
#
|
|
33
30
|
# @param after_cursor [String] Cursor for pagination
|
|
34
31
|
#
|
|
35
32
|
# @param limit [Integer] Maximum number of items to return (1-100, default 50). For paid per-result endpo
|
|
@@ -30,9 +30,6 @@ module XTwitterScraper
|
|
|
30
30
|
optional :type, enum: -> { XTwitterScraper::DrawExportParams::Type }
|
|
31
31
|
|
|
32
32
|
# @!method initialize(id:, format_:, type: nil, request_options: {})
|
|
33
|
-
# Some parameter documentations has been truncated, see
|
|
34
|
-
# {XTwitterScraper::Models::DrawExportParams} for more details.
|
|
35
|
-
#
|
|
36
33
|
# @param id [String]
|
|
37
34
|
#
|
|
38
35
|
# @param format_ [Symbol, XTwitterScraper::Models::DrawExportParams::Format] Export output format. PDF entry exports include up to 10,000 rows. Other entry f
|
|
@@ -27,9 +27,6 @@ module XTwitterScraper
|
|
|
27
27
|
optional :limit, Integer
|
|
28
28
|
|
|
29
29
|
# @!method initialize(cursor: nil, limit: nil, request_options: {})
|
|
30
|
-
# Some parameter documentations has been truncated, see
|
|
31
|
-
# {XTwitterScraper::Models::DrawListParams} for more details.
|
|
32
|
-
#
|
|
33
30
|
# @param cursor [String] Previous nextCursor.
|
|
34
31
|
#
|
|
35
32
|
# @param limit [Integer] Maximum number of items to return (1-100, default 50). For paid per-result endpo
|
|
@@ -38,9 +38,6 @@ module XTwitterScraper
|
|
|
38
38
|
required :winners, -> { XTwitterScraper::Internal::Type::ArrayOf[XTwitterScraper::Winner] }
|
|
39
39
|
|
|
40
40
|
# @!method initialize(id:, total_entries:, tweet_id:, valid_entries:, winners:)
|
|
41
|
-
# Some parameter documentations has been truncated, see
|
|
42
|
-
# {XTwitterScraper::Models::DrawRunResponse} for more details.
|
|
43
|
-
#
|
|
44
41
|
# @param id [String]
|
|
45
42
|
#
|
|
46
43
|
# @param total_entries [Integer] Candidate entries inspected for this draw after the credit-derived cap. This may
|