bad_pigeon 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/exe/pigeon +9 -0
- data/lib/bad_pigeon/elements/component.rb +26 -0
- data/lib/bad_pigeon/elements/timeline_entry.rb +56 -0
- data/lib/bad_pigeon/elements/timeline_instruction.rb +57 -0
- data/lib/bad_pigeon/elements/timeline_tweet.rb +34 -0
- data/lib/bad_pigeon/entry_filter.rb +32 -0
- data/lib/bad_pigeon/har/har_archive.rb +14 -0
- data/lib/bad_pigeon/har/har_request.rb +41 -0
- data/lib/bad_pigeon/models/tweet.rb +99 -0
- data/lib/bad_pigeon/models/url_entity.rb +13 -0
- data/lib/bad_pigeon/models/user.rb +17 -0
- data/lib/bad_pigeon/timelines/home_timeline.rb +22 -0
- data/lib/bad_pigeon/timelines/list_timeline.rb +22 -0
- data/lib/bad_pigeon/timelines/user_timeline.rb +26 -0
- data/lib/bad_pigeon/timelines.rb +10 -0
- data/lib/bad_pigeon/tweet_extractor.rb +37 -0
- data/lib/bad_pigeon/util/assertions.rb +23 -0
- data/lib/bad_pigeon/util/strict_hash.rb +16 -0
- data/lib/bad_pigeon/version.rb +5 -0
- data/lib/bad_pigeon.rb +9 -0
- data/sig/bad_pigeon.rbs +4 -0
- metadata +92 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 383a6c54b644b03f93d3b6c67ca811045938f8bf0de44feaf3c5f7344838eaf8
|
4
|
+
data.tar.gz: b69a8b24eeb45f7e3761309aa993620ed845a6625ad8fbaae1114c398578e17c
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 35ccd0f949bca44ce2b0374f1ba185f7addedb35c2b574e3bedfbccf9c7487ca80a3d418f0e9cd27c8501ee25c874d564d77f1959e1a8e9a4925707673861be9
|
7
|
+
data.tar.gz: 1af6038dafd3046b4f3af1cfcb739dd1df4428d527b0eea1ad986ef913711e0b67d9ce1ea8148d3f4d58b0d73b940e93409887f604c526c8c49c655d81608b5a
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
The zlib License
|
2
|
+
|
3
|
+
Copyright (c) 2023 Jakub Suder
|
4
|
+
|
5
|
+
This software is provided 'as-is', without any express or implied
|
6
|
+
warranty. In no event will the authors be held liable for any damages
|
7
|
+
arising from the use of this software.
|
8
|
+
|
9
|
+
Permission is granted to anyone to use this software for any purpose,
|
10
|
+
including commercial applications, and to alter it and redistribute it
|
11
|
+
freely, subject to the following restrictions:
|
12
|
+
|
13
|
+
1. The origin of this software must not be misrepresented; you must not
|
14
|
+
claim that you wrote the original software. If you use this software
|
15
|
+
in a product, an acknowledgment in the product documentation would be
|
16
|
+
appreciated but is not required.
|
17
|
+
|
18
|
+
2. Altered source versions must be plainly marked as such, and must not be
|
19
|
+
misrepresented as being the original software.
|
20
|
+
|
21
|
+
3. This notice may not be removed or altered from any source distribution.
|
22
|
+
|
data/README.md
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# BadPigeon
|
2
|
+
|
3
|
+
TODO: Delete this and the text below, and describe your gem
|
4
|
+
|
5
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bad_pigeon`. To experiment with that code, run `bin/console` for an interactive prompt.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
|
10
|
+
|
11
|
+
Install the gem and add to the application's Gemfile by executing:
|
12
|
+
|
13
|
+
$ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
|
14
|
+
|
15
|
+
If bundler is not being used to manage dependencies, install the gem by executing:
|
16
|
+
|
17
|
+
$ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
|
18
|
+
|
19
|
+
## Usage
|
20
|
+
|
21
|
+
TODO: Write usage instructions here
|
22
|
+
|
23
|
+
## Development
|
24
|
+
|
25
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
26
|
+
|
27
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
28
|
+
|
29
|
+
## Contributing
|
30
|
+
|
31
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bad_pigeon.
|
data/exe/pigeon
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
module BadPigeon
|
2
|
+
module Component
|
3
|
+
# normal tweet in e.g. home/latest or a user's timeline
|
4
|
+
ORGANIC_FEED_TWEET = "suggest_ranked_organic_tweet"
|
5
|
+
|
6
|
+
# tweet in a list timeline
|
7
|
+
ORGANIC_LIST_TWEET = "suggest_organic_list_tweet"
|
8
|
+
|
9
|
+
# user's pinned tweet
|
10
|
+
PINNED_TWEET = "suggest_pinned_tweet"
|
11
|
+
|
12
|
+
# reply that shows up in your timeline
|
13
|
+
EXTENDED_REPLY = "suggest_extended_reply"
|
14
|
+
|
15
|
+
# algorithmic timeline suggested tweets
|
16
|
+
SOCIAL_CONTEXT = "suggest_sc_tweet" # X follows
|
17
|
+
SOCIAL_ACTIVITY = "suggest_activity_tweet" # X liked
|
18
|
+
RANKED_FEED_TWEET = "suggest_ranked_timeline_tweet"
|
19
|
+
|
20
|
+
# promoted tweet (ad)
|
21
|
+
PROMOTED_TWEET = "suggest_promoted"
|
22
|
+
|
23
|
+
# "Who to follow" block
|
24
|
+
FOLLOW_SUGGESTIONS = "suggest_who_to_follow"
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'bad_pigeon/elements/timeline_tweet'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class TimelineEntry
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
module Type
|
9
|
+
ITEM = "TimelineTimelineItem"
|
10
|
+
MODULE = "TimelineTimelineModule"
|
11
|
+
CURSOR = "TimelineTimelineCursor"
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_reader :json
|
15
|
+
|
16
|
+
def initialize(json)
|
17
|
+
@json = json
|
18
|
+
|
19
|
+
assert { json['entryType'] == json['__typename'] }
|
20
|
+
end
|
21
|
+
|
22
|
+
def type
|
23
|
+
@json['entryType']
|
24
|
+
end
|
25
|
+
|
26
|
+
def component
|
27
|
+
@json['clientEventInfo'] && @json['clientEventInfo']['component']
|
28
|
+
end
|
29
|
+
|
30
|
+
def items
|
31
|
+
case self.type
|
32
|
+
when Type::ITEM
|
33
|
+
item_from_content(@json['itemContent'])
|
34
|
+
when Type::MODULE
|
35
|
+
@json['items'].map { |i| item_from_content(i['item']['itemContent']) }
|
36
|
+
when Type::CURSOR
|
37
|
+
[]
|
38
|
+
else
|
39
|
+
assert("Unknown entry type: #{type}")
|
40
|
+
[]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def item_from_content(item_content)
|
45
|
+
case item_content['itemType']
|
46
|
+
when 'TimelineTweet'
|
47
|
+
[TimelineTweet.new(item_content)]
|
48
|
+
when 'TimelineUser'
|
49
|
+
[]
|
50
|
+
else
|
51
|
+
assert("Unknown itemContent type: #{item_content['itemType']}")
|
52
|
+
[]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'bad_pigeon/elements/timeline_entry'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class TimelineInstruction
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
module Type
|
9
|
+
ADD_ENTRIES = "TimelineAddEntries"
|
10
|
+
CLEAR_CACHE = "TimelineClearCache"
|
11
|
+
PIN_ENTRY = "TimelinePinEntry"
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(json, timeline_class)
|
15
|
+
@json = json
|
16
|
+
@timeline_class = timeline_class
|
17
|
+
end
|
18
|
+
|
19
|
+
def type
|
20
|
+
@json['type']
|
21
|
+
end
|
22
|
+
|
23
|
+
def entries
|
24
|
+
case type
|
25
|
+
when Type::ADD_ENTRIES
|
26
|
+
expected_keys = ['entries', 'type']
|
27
|
+
entries = @json['entries']
|
28
|
+
|
29
|
+
case @timeline_class
|
30
|
+
when ListTimeline
|
31
|
+
expected_types = [TimelineEntry::Type::ITEM, TimelineEntry::Type::CURSOR]
|
32
|
+
else
|
33
|
+
expected_types = [TimelineEntry::Type::ITEM, TimelineEntry::Type::MODULE, TimelineEntry::Type::CURSOR]
|
34
|
+
end
|
35
|
+
|
36
|
+
when Type::CLEAR_CACHE
|
37
|
+
expected_keys = ['type']
|
38
|
+
entries = []
|
39
|
+
|
40
|
+
when Type::PIN_ENTRY
|
41
|
+
expected_keys = ['entry', 'type']
|
42
|
+
entries = [@json['entry']]
|
43
|
+
expected_types = [TimelineEntry::Type::ITEM]
|
44
|
+
|
45
|
+
else
|
46
|
+
assert("Unknown timeline instruction type: #{type}")
|
47
|
+
return []
|
48
|
+
end
|
49
|
+
|
50
|
+
assert { @json.keys.sort == expected_keys }
|
51
|
+
assert { entries.all? { |e| expected_types.include?(e['content']['entryType']) }}
|
52
|
+
assert { entries.all? { |e| e.keys.sort == ['content', 'entryId', 'sortIndex'] }}
|
53
|
+
|
54
|
+
entries.map { |e| TimelineEntry.new(e['content']) }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'bad_pigeon/models/tweet'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class TimelineTweet
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
def initialize(json)
|
9
|
+
@json = json
|
10
|
+
end
|
11
|
+
|
12
|
+
def result_type
|
13
|
+
@json['tweet_results']['result'] && @json['tweet_results']['result']['__typename']
|
14
|
+
end
|
15
|
+
|
16
|
+
def tweet_data
|
17
|
+
case result_type
|
18
|
+
when 'Tweet', 'TweetWithVisibilityResults'
|
19
|
+
@json['tweet_results']['result']
|
20
|
+
when 'TweetUnavailable'
|
21
|
+
nil
|
22
|
+
when nil
|
23
|
+
nil
|
24
|
+
else
|
25
|
+
assert("Unknown tweet result type: #{result_type}")
|
26
|
+
nil
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def tweet
|
31
|
+
Tweet.new(tweet_data)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require_relative 'elements/component'
|
2
|
+
require_relative 'util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class EntryFilter
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
def include_entry?(entry)
|
9
|
+
case entry.component
|
10
|
+
when Component::ORGANIC_FEED_TWEET,
|
11
|
+
Component::ORGANIC_LIST_TWEET,
|
12
|
+
Component::PINNED_TWEET,
|
13
|
+
Component::EXTENDED_REPLY,
|
14
|
+
Component::SOCIAL_CONTEXT,
|
15
|
+
Component::SOCIAL_ACTIVITY,
|
16
|
+
Component::RANKED_FEED_TWEET
|
17
|
+
then true
|
18
|
+
|
19
|
+
when Component::PROMOTED_TWEET,
|
20
|
+
Component::FOLLOW_SUGGESTIONS
|
21
|
+
then false
|
22
|
+
|
23
|
+
when nil
|
24
|
+
true
|
25
|
+
|
26
|
+
else
|
27
|
+
assert("Unknown component: #{entry.component}")
|
28
|
+
false
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module BadPigeon
|
4
|
+
class HARRequest
|
5
|
+
def initialize(json)
|
6
|
+
@json = json
|
7
|
+
end
|
8
|
+
|
9
|
+
def method
|
10
|
+
@json['request']['method'].downcase.to_sym
|
11
|
+
end
|
12
|
+
|
13
|
+
def url
|
14
|
+
@json['request']['url']
|
15
|
+
end
|
16
|
+
|
17
|
+
def graphql_endpoint?
|
18
|
+
url.start_with?('https://api.twitter.com/graphql/') || url.start_with?('https://twitter.com/i/api/graphql/')
|
19
|
+
end
|
20
|
+
|
21
|
+
def status
|
22
|
+
@json['response']['status']
|
23
|
+
end
|
24
|
+
|
25
|
+
def mime_type
|
26
|
+
@json['response']['content']['mimeType']
|
27
|
+
end
|
28
|
+
|
29
|
+
def has_json_response?
|
30
|
+
mime_type == 'application/json'
|
31
|
+
end
|
32
|
+
|
33
|
+
def response_body
|
34
|
+
@json['response']['content']['text']
|
35
|
+
end
|
36
|
+
|
37
|
+
def response_json
|
38
|
+
response_body && JSON.parse(response_body)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'bad_pigeon/models/url_entity'
|
2
|
+
require 'bad_pigeon/models/user'
|
3
|
+
require 'bad_pigeon/util/assertions'
|
4
|
+
require 'bad_pigeon/util/strict_hash'
|
5
|
+
|
6
|
+
require 'time'
|
7
|
+
|
8
|
+
module BadPigeon
|
9
|
+
class Tweet
|
10
|
+
include Assertions
|
11
|
+
|
12
|
+
attr_reader :json
|
13
|
+
|
14
|
+
def initialize(json)
|
15
|
+
case json['__typename']
|
16
|
+
when 'Tweet'
|
17
|
+
@json = json
|
18
|
+
when 'TweetWithVisibilityResults'
|
19
|
+
@json = json['tweet']
|
20
|
+
else
|
21
|
+
assert("Unexpected tweet record type: #{json['__typename']}")
|
22
|
+
@json = json['tweet']
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def legacy
|
27
|
+
json['legacy']
|
28
|
+
end
|
29
|
+
|
30
|
+
def id
|
31
|
+
legacy['id_str'].to_i
|
32
|
+
end
|
33
|
+
|
34
|
+
def user
|
35
|
+
User.new(json['core']['user_results']['result'])
|
36
|
+
end
|
37
|
+
|
38
|
+
def created_at
|
39
|
+
Time.parse(legacy['created_at'])
|
40
|
+
end
|
41
|
+
|
42
|
+
def text
|
43
|
+
legacy['full_text']
|
44
|
+
end
|
45
|
+
|
46
|
+
def retweet?
|
47
|
+
!!legacy['retweeted_status_result']
|
48
|
+
end
|
49
|
+
|
50
|
+
alias retweeted_status? retweet?
|
51
|
+
|
52
|
+
def retweeted_status
|
53
|
+
legacy['retweeted_status_result'] && Tweet.new(legacy['retweeted_status_result']['result'])
|
54
|
+
end
|
55
|
+
|
56
|
+
def quoted_status?
|
57
|
+
# there is also legacy['is_quote_status'], but it may be true while quoted_status_result
|
58
|
+
# is not set if the quoted status was deleted
|
59
|
+
!!json['quoted_status_result']
|
60
|
+
end
|
61
|
+
|
62
|
+
def quoted_status
|
63
|
+
json['quoted_status_result'] && Tweet.new(json['quoted_status_result']['result'])
|
64
|
+
end
|
65
|
+
|
66
|
+
alias quoted_tweet? quoted_status?
|
67
|
+
alias quoted_tweet quoted_status
|
68
|
+
|
69
|
+
def urls
|
70
|
+
legacy['entities']['urls'].map { |u| URLEntity.new(u) }
|
71
|
+
end
|
72
|
+
|
73
|
+
def attrs
|
74
|
+
user = json['core']['user_results']['result']
|
75
|
+
|
76
|
+
fields = legacy.merge({
|
77
|
+
id: id,
|
78
|
+
source: json['source'],
|
79
|
+
text: text,
|
80
|
+
truncated: false,
|
81
|
+
}).reject { |k, v| ['retweeted_status_result', 'quoted_status_result'].include?(k) }
|
82
|
+
|
83
|
+
user_fields = user['legacy'].merge({
|
84
|
+
id: user['rest_id'].to_i,
|
85
|
+
id_str: user['rest_id'],
|
86
|
+
}).reject { |k, v| k =~ /^profile_\w+_extensions/ }
|
87
|
+
|
88
|
+
fields[:user] = StrictHash[user_fields]
|
89
|
+
|
90
|
+
if quoted_status?
|
91
|
+
fields[:quoted_status] = quoted_status.attrs
|
92
|
+
elsif retweeted_status?
|
93
|
+
fields[:retweeted_status] = retweeted_status.attrs
|
94
|
+
end
|
95
|
+
|
96
|
+
StrictHash[fields]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'bad_pigeon/elements/timeline_instruction'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class HomeTimeline
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
EXPECTED_INSTRUCTIONS = [TimelineInstruction::Type::ADD_ENTRIES]
|
9
|
+
|
10
|
+
def initialize(json)
|
11
|
+
@json = json
|
12
|
+
end
|
13
|
+
|
14
|
+
def instructions
|
15
|
+
@instructions ||= begin
|
16
|
+
list = @json['data']['home']['home_timeline_urt']['instructions']
|
17
|
+
assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
|
18
|
+
list.map { |j| TimelineInstruction.new(j, self.class) }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'bad_pigeon/elements/timeline_instruction'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class ListTimeline
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
EXPECTED_INSTRUCTIONS = [TimelineInstruction::Type::ADD_ENTRIES]
|
9
|
+
|
10
|
+
def initialize(json)
|
11
|
+
@json = json
|
12
|
+
end
|
13
|
+
|
14
|
+
def instructions
|
15
|
+
@instructions ||= begin
|
16
|
+
list = @json['data']['list']['tweets_timeline']['timeline']['instructions']
|
17
|
+
assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
|
18
|
+
list.map { |j| TimelineInstruction.new(j, self.class) }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'bad_pigeon/elements/timeline_instruction'
|
2
|
+
require 'bad_pigeon/util/assertions'
|
3
|
+
|
4
|
+
module BadPigeon
|
5
|
+
class UserTimeline
|
6
|
+
include Assertions
|
7
|
+
|
8
|
+
EXPECTED_INSTRUCTIONS = [
|
9
|
+
TimelineInstruction::Type::ADD_ENTRIES,
|
10
|
+
TimelineInstruction::Type::CLEAR_CACHE,
|
11
|
+
TimelineInstruction::Type::PIN_ENTRY
|
12
|
+
]
|
13
|
+
|
14
|
+
def initialize(json)
|
15
|
+
@json = json
|
16
|
+
end
|
17
|
+
|
18
|
+
def instructions
|
19
|
+
@instructions ||= begin
|
20
|
+
list = @json['data']['user']['result']['timeline_v2']['timeline']['instructions']
|
21
|
+
assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
|
22
|
+
list.map { |j| TimelineInstruction.new(j, self.class) }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
Dir[File.join(__dir__, 'timelines', '*.rb')].each { |f| require(f) }
|
2
|
+
|
3
|
+
module BadPigeon
|
4
|
+
TIMELINE_TYPES = {
|
5
|
+
'UserTweets' => UserTimeline,
|
6
|
+
'HomeLatestTimeline' => HomeTimeline,
|
7
|
+
'HomeTimeline' => HomeTimeline,
|
8
|
+
'ListLatestTweetsTimeline' => ListTimeline
|
9
|
+
}
|
10
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative 'har/har_archive'
|
2
|
+
require_relative 'entry_filter'
|
3
|
+
require_relative 'util/assertions'
|
4
|
+
require_relative 'timelines'
|
5
|
+
|
6
|
+
require 'uri'
|
7
|
+
|
8
|
+
module BadPigeon
|
9
|
+
class TweetExtractor
|
10
|
+
include Assertions
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@filter = EntryFilter.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_tweets_from_har(har_data)
|
17
|
+
archive = HARArchive.new(har_data)
|
18
|
+
|
19
|
+
requests = archive.entries.select { |e|
|
20
|
+
e.graphql_endpoint? && e.method == :get && e.status == 200 && e.has_json_response?
|
21
|
+
}
|
22
|
+
|
23
|
+
entries = requests.map { |e|
|
24
|
+
endpoint = URI(e.url).path.split('/').last
|
25
|
+
|
26
|
+
if timeline_class = TIMELINE_TYPES[endpoint]
|
27
|
+
timeline_class.new(e.response_json).instructions.map(&:entries)
|
28
|
+
elsif !TIMELINE_TYPES.has_key?(endpoint)
|
29
|
+
debug "Unknown endpoint: #{endpoint}"
|
30
|
+
[]
|
31
|
+
end
|
32
|
+
}.flatten
|
33
|
+
|
34
|
+
entries.select { |e| @filter.include_entry?(e) }.map(&:items).flatten.map(&:tweet).compact
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module AngryPigeon
|
2
|
+
class AssertionError < StandardError; end
|
3
|
+
end
|
4
|
+
|
5
|
+
module BadPigeon
|
6
|
+
module Assertions
|
7
|
+
def self.included(target)
|
8
|
+
if ENV['ANGRY_PIGEON'] == '1'
|
9
|
+
target.define_method(:assert) do |msg = nil, &block|
|
10
|
+
raise AngryPigeon::AssertionError.new(msg) if msg || block.call == false
|
11
|
+
end
|
12
|
+
target.define_method(:debug) do |msg|
|
13
|
+
$stderr.puts msg
|
14
|
+
end
|
15
|
+
else
|
16
|
+
target.define_method(:assert) do |msg = nil, &block|
|
17
|
+
end
|
18
|
+
target.define_method(:debug) do |msg|
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/bad_pigeon.rb
ADDED
data/sig/bad_pigeon.rbs
ADDED
metadata
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: bad_pigeon
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kuba Suder
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2023-06-19 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: addressable
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.8'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.8'
|
27
|
+
description: "\n BadPigeon is a Ruby gem that allows you to extract tweet data
|
28
|
+
from the XHR requests that the Twitter.com frontend\n website does in user's
|
29
|
+
browser. The requests need to be saved into a \"HAR\" archive file from the browser's
|
30
|
+
web\n inspector tool and then that file is fed into either the appropriate Ruby
|
31
|
+
class or the `pigeon` command line tool.\n \n The tool intents to be API compatible
|
32
|
+
with the popular `twitter` gem and generate the same kind of tweet JSON\n structure
|
33
|
+
as is read and exported by that library.\n "
|
34
|
+
email:
|
35
|
+
- jakub.suder@gmail.com
|
36
|
+
executables:
|
37
|
+
- pigeon
|
38
|
+
extensions: []
|
39
|
+
extra_rdoc_files: []
|
40
|
+
files:
|
41
|
+
- CHANGELOG.md
|
42
|
+
- LICENSE.txt
|
43
|
+
- README.md
|
44
|
+
- exe/pigeon
|
45
|
+
- lib/bad_pigeon.rb
|
46
|
+
- lib/bad_pigeon/elements/component.rb
|
47
|
+
- lib/bad_pigeon/elements/timeline_entry.rb
|
48
|
+
- lib/bad_pigeon/elements/timeline_instruction.rb
|
49
|
+
- lib/bad_pigeon/elements/timeline_tweet.rb
|
50
|
+
- lib/bad_pigeon/entry_filter.rb
|
51
|
+
- lib/bad_pigeon/har/har_archive.rb
|
52
|
+
- lib/bad_pigeon/har/har_request.rb
|
53
|
+
- lib/bad_pigeon/models/tweet.rb
|
54
|
+
- lib/bad_pigeon/models/url_entity.rb
|
55
|
+
- lib/bad_pigeon/models/user.rb
|
56
|
+
- lib/bad_pigeon/timelines.rb
|
57
|
+
- lib/bad_pigeon/timelines/home_timeline.rb
|
58
|
+
- lib/bad_pigeon/timelines/list_timeline.rb
|
59
|
+
- lib/bad_pigeon/timelines/user_timeline.rb
|
60
|
+
- lib/bad_pigeon/tweet_extractor.rb
|
61
|
+
- lib/bad_pigeon/util/assertions.rb
|
62
|
+
- lib/bad_pigeon/util/strict_hash.rb
|
63
|
+
- lib/bad_pigeon/version.rb
|
64
|
+
- sig/bad_pigeon.rbs
|
65
|
+
homepage: https://github.com/mackuba/bad_pigeon
|
66
|
+
licenses:
|
67
|
+
- Zlib
|
68
|
+
metadata:
|
69
|
+
bug_tracker_uri: https://github.com/mackuba/bad_pigeon/issues
|
70
|
+
changelog_uri: https://github.com/mackuba/bad_pigeon/blob/master/CHANGELOG.md
|
71
|
+
source_code_uri: https://github.com/mackuba/bad_pigeon
|
72
|
+
post_install_message:
|
73
|
+
rdoc_options: []
|
74
|
+
require_paths:
|
75
|
+
- lib
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: 2.6.0
|
81
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
requirements: []
|
87
|
+
rubygems_version: 3.4.10
|
88
|
+
signing_key:
|
89
|
+
specification_version: 4
|
90
|
+
summary: A tool for extracting tweet data from GraphQL fetch requests made by the
|
91
|
+
Twitter website
|
92
|
+
test_files: []
|