bad_pigeon 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 383a6c54b644b03f93d3b6c67ca811045938f8bf0de44feaf3c5f7344838eaf8
+  data.tar.gz: b69a8b24eeb45f7e3761309aa993620ed845a6625ad8fbaae1114c398578e17c
+SHA512:
+  metadata.gz: 35ccd0f949bca44ce2b0374f1ba185f7addedb35c2b574e3bedfbccf9c7487ca80a3d418f0e9cd27c8501ee25c874d564d77f1959e1a8e9a4925707673861be9
+  data.tar.gz: 1af6038dafd3046b4f3af1cfcb739dd1df4428d527b0eea1ad986ef913711e0b67d9ce1ea8148d3f4d58b0d73b940e93409887f604c526c8c49c655d81608b5a

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-06-18
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+The zlib License
+
+Copyright (c) 2023 Jakub Suder
+
+This software is provided 'as-is', without any express or implied
+warranty.  In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+3. This notice may not be removed or altered from any source distribution.
+

data/README.md

@@ -0,0 +1,31 @@
+# BadPigeon
+
+TODO: Delete this and the text below, and describe your gem
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bad_pigeon`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bad_pigeon.

data/exe/pigeon

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'bad_pigeon'
+require 'json'
+
+extractor = BadPigeon::TweetExtractor.new
+archive_data = STDIN.read
+tweets = extractor.get_tweets_from_har(archive_data)
+puts JSON.generate(tweets.map(&:attrs))

data/lib/bad_pigeon/elements/component.rb

@@ -0,0 +1,26 @@
+module BadPigeon
+  module Component
+    # normal tweet in e.g. home/latest or a user's timeline
+    ORGANIC_FEED_TWEET = "suggest_ranked_organic_tweet"
+
+    # tweet in a list timeline
+    ORGANIC_LIST_TWEET = "suggest_organic_list_tweet"
+
+    # user's pinned tweet
+    PINNED_TWEET = "suggest_pinned_tweet"
+
+    # reply that shows up in your timeline
+    EXTENDED_REPLY = "suggest_extended_reply"
+
+    # algorithmic timeline suggested tweets
+    SOCIAL_CONTEXT = "suggest_sc_tweet"                     # X follows
+    SOCIAL_ACTIVITY = "suggest_activity_tweet"              # X liked
+    RANKED_FEED_TWEET = "suggest_ranked_timeline_tweet"
+
+    # promoted tweet (ad)
+    PROMOTED_TWEET = "suggest_promoted"
+
+    # "Who to follow" block
+    FOLLOW_SUGGESTIONS = "suggest_who_to_follow"
+  end
+end

data/lib/bad_pigeon/elements/timeline_entry.rb

@@ -0,0 +1,56 @@
+require 'bad_pigeon/elements/timeline_tweet'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class TimelineEntry
+    include Assertions
+
+    module Type
+      ITEM = "TimelineTimelineItem"
+      MODULE = "TimelineTimelineModule"
+      CURSOR = "TimelineTimelineCursor"
+    end
+
+    attr_reader :json
+
+    def initialize(json)
+      @json = json
+
+      assert { json['entryType'] == json['__typename'] }
+    end
+
+    def type
+      @json['entryType']
+    end
+
+    def component
+      @json['clientEventInfo'] && @json['clientEventInfo']['component']
+    end
+
+    def items
+      case self.type
+      when Type::ITEM
+        item_from_content(@json['itemContent'])
+      when Type::MODULE
+        @json['items'].map { |i| item_from_content(i['item']['itemContent']) }
+      when Type::CURSOR
+        []
+      else
+        assert("Unknown entry type: #{type}")
+        []
+      end
+    end
+
+    def item_from_content(item_content)
+      case item_content['itemType']
+      when 'TimelineTweet'
+        [TimelineTweet.new(item_content)]
+      when 'TimelineUser'
+        []
+      else
+        assert("Unknown itemContent type: #{item_content['itemType']}")
+        []
+      end
+    end
+  end
+end

data/lib/bad_pigeon/elements/timeline_instruction.rb

@@ -0,0 +1,57 @@
+require 'bad_pigeon/elements/timeline_entry'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class TimelineInstruction
+    include Assertions
+
+    module Type
+      ADD_ENTRIES = "TimelineAddEntries"
+      CLEAR_CACHE = "TimelineClearCache"
+      PIN_ENTRY = "TimelinePinEntry"
+    end
+
+    def initialize(json, timeline_class)
+      @json = json
+      @timeline_class = timeline_class
+    end
+
+    def type
+      @json['type']
+    end
+
+    def entries
+      case type
+      when Type::ADD_ENTRIES
+        expected_keys = ['entries', 'type']
+        entries = @json['entries']
+
+        case @timeline_class
+        when ListTimeline
+          expected_types = [TimelineEntry::Type::ITEM, TimelineEntry::Type::CURSOR]
+        else
+          expected_types = [TimelineEntry::Type::ITEM, TimelineEntry::Type::MODULE, TimelineEntry::Type::CURSOR]
+        end
+
+      when Type::CLEAR_CACHE
+        expected_keys = ['type']
+        entries = []
+
+      when Type::PIN_ENTRY
+        expected_keys = ['entry', 'type']
+        entries = [@json['entry']]
+        expected_types = [TimelineEntry::Type::ITEM]
+
+      else
+        assert("Unknown timeline instruction type: #{type}")
+        return []
+      end
+
+      assert { @json.keys.sort == expected_keys }
+      assert { entries.all? { |e| expected_types.include?(e['content']['entryType']) }}
+      assert { entries.all? { |e| e.keys.sort == ['content', 'entryId', 'sortIndex'] }}
+
+      entries.map { |e| TimelineEntry.new(e['content']) }
+    end
+  end
+end

data/lib/bad_pigeon/elements/timeline_tweet.rb

@@ -0,0 +1,34 @@
+require 'bad_pigeon/models/tweet'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class TimelineTweet
+    include Assertions
+
+    def initialize(json)
+      @json = json
+    end
+
+    def result_type
+      @json['tweet_results']['result'] && @json['tweet_results']['result']['__typename']
+    end
+
+    def tweet_data
+      case result_type
+      when 'Tweet', 'TweetWithVisibilityResults'
+        @json['tweet_results']['result']
+      when 'TweetUnavailable'
+        nil
+      when nil
+        nil
+      else
+        assert("Unknown tweet result type: #{result_type}")
+        nil
+      end
+    end
+
+    def tweet
+      Tweet.new(tweet_data)
+    end
+  end
+end

data/lib/bad_pigeon/entry_filter.rb

@@ -0,0 +1,32 @@
+require_relative 'elements/component'
+require_relative 'util/assertions'
+
+module BadPigeon
+  class EntryFilter
+    include Assertions
+
+    def include_entry?(entry)
+      case entry.component
+      when Component::ORGANIC_FEED_TWEET,
+           Component::ORGANIC_LIST_TWEET,
+           Component::PINNED_TWEET,
+           Component::EXTENDED_REPLY,
+           Component::SOCIAL_CONTEXT,
+           Component::SOCIAL_ACTIVITY,
+           Component::RANKED_FEED_TWEET
+           then true
+
+      when Component::PROMOTED_TWEET,
+           Component::FOLLOW_SUGGESTIONS
+           then false
+
+      when nil
+        true
+
+      else
+        assert("Unknown component: #{entry.component}")
+        false
+      end
+    end
+  end
+end

data/lib/bad_pigeon/har/har_archive.rb

@@ -0,0 +1,14 @@
+require_relative 'har_request'
+require 'json'
+
+module BadPigeon
+  class HARArchive
+    def initialize(data)
+      @json = JSON.parse(data)
+    end
+
+    def entries
+      @json['log']['entries'].map { |j| HARRequest.new(j) }
+    end
+  end
+end

data/lib/bad_pigeon/har/har_request.rb

@@ -0,0 +1,41 @@
+require 'json'
+
+module BadPigeon
+  class HARRequest
+    def initialize(json)
+      @json = json
+    end
+
+    def method
+      @json['request']['method'].downcase.to_sym
+    end
+
+    def url
+      @json['request']['url']
+    end
+
+    def graphql_endpoint?
+      url.start_with?('https://api.twitter.com/graphql/') || url.start_with?('https://twitter.com/i/api/graphql/')
+    end
+
+    def status
+      @json['response']['status']
+    end
+
+    def mime_type
+      @json['response']['content']['mimeType']
+    end
+
+    def has_json_response?
+      mime_type == 'application/json'
+    end
+
+    def response_body
+      @json['response']['content']['text']
+    end
+
+    def response_json
+      response_body && JSON.parse(response_body)
+    end
+  end
+end

data/lib/bad_pigeon/models/tweet.rb

@@ -0,0 +1,99 @@
+require 'bad_pigeon/models/url_entity'
+require 'bad_pigeon/models/user'
+require 'bad_pigeon/util/assertions'
+require 'bad_pigeon/util/strict_hash'
+
+require 'time'
+
+module BadPigeon
+  class Tweet
+    include Assertions
+
+    attr_reader :json
+
+    def initialize(json)
+      case json['__typename']
+      when 'Tweet'
+        @json = json
+      when 'TweetWithVisibilityResults'
+        @json = json['tweet']
+      else
+        assert("Unexpected tweet record type: #{json['__typename']}")
+        @json = json['tweet']
+      end
+    end
+
+    def legacy
+      json['legacy']
+    end
+
+    def id
+      legacy['id_str'].to_i
+    end
+
+    def user
+      User.new(json['core']['user_results']['result'])
+    end
+
+    def created_at
+      Time.parse(legacy['created_at'])
+    end
+
+    def text
+      legacy['full_text']
+    end
+
+    def retweet?
+      !!legacy['retweeted_status_result']
+    end
+
+    alias retweeted_status? retweet?
+
+    def retweeted_status
+      legacy['retweeted_status_result'] && Tweet.new(legacy['retweeted_status_result']['result'])
+    end
+
+    def quoted_status?
+      # there is also legacy['is_quote_status'], but it may be true while quoted_status_result
+      # is not set if the quoted status was deleted
+      !!json['quoted_status_result']
+    end
+
+    def quoted_status
+      json['quoted_status_result'] && Tweet.new(json['quoted_status_result']['result'])
+    end
+
+    alias quoted_tweet? quoted_status?
+    alias quoted_tweet quoted_status
+
+    def urls
+      legacy['entities']['urls'].map { |u| URLEntity.new(u) }
+    end
+
+    def attrs
+      user = json['core']['user_results']['result']
+
+      fields = legacy.merge({
+        id: id,
+        source: json['source'],
+        text: text,
+        truncated: false,
+      }).reject { |k, v| ['retweeted_status_result', 'quoted_status_result'].include?(k) }
+
+      user_fields = user['legacy'].merge({
+        id: user['rest_id'].to_i,
+        id_str: user['rest_id'],
+      }).reject { |k, v| k =~ /^profile_\w+_extensions/ }
+
+      fields[:user] = StrictHash[user_fields]
+
+      if quoted_status?
+        fields[:quoted_status] = quoted_status.attrs
+      elsif retweeted_status?
+        fields[:retweeted_status] = retweeted_status.attrs
+      end
+
+      StrictHash[fields]
+    end
+  end
+end

data/lib/bad_pigeon/models/url_entity.rb

@@ -0,0 +1,13 @@
+require 'addressable/uri'
+
+module BadPigeon
+  class URLEntity
+    def initialize(json)
+      @json = json
+    end
+
+    def expanded_url
+      Addressable::URI.parse(@json['expanded_url'])
+    end
+  end
+end

data/lib/bad_pigeon/models/user.rb

@@ -0,0 +1,17 @@
+module BadPigeon
+  class User
+    attr_reader :json
+
+    def initialize(json)
+      @json = json
+    end
+
+    def legacy
+      json['legacy']
+    end
+
+    def screen_name
+      legacy['screen_name']
+    end
+  end
+end

data/lib/bad_pigeon/timelines/home_timeline.rb

@@ -0,0 +1,22 @@
+require 'bad_pigeon/elements/timeline_instruction'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class HomeTimeline
+    include Assertions
+
+    EXPECTED_INSTRUCTIONS = [TimelineInstruction::Type::ADD_ENTRIES]
+
+    def initialize(json)
+      @json = json
+    end
+
+    def instructions
+      @instructions ||= begin
+        list = @json['data']['home']['home_timeline_urt']['instructions']
+        assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
+        list.map { |j| TimelineInstruction.new(j, self.class) }
+      end
+    end
+  end
+end

data/lib/bad_pigeon/timelines/list_timeline.rb

@@ -0,0 +1,22 @@
+require 'bad_pigeon/elements/timeline_instruction'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class ListTimeline
+    include Assertions
+
+    EXPECTED_INSTRUCTIONS = [TimelineInstruction::Type::ADD_ENTRIES]
+
+    def initialize(json)
+      @json = json
+    end
+
+    def instructions
+      @instructions ||= begin
+        list = @json['data']['list']['tweets_timeline']['timeline']['instructions']
+        assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
+        list.map { |j| TimelineInstruction.new(j, self.class) }
+      end
+    end
+  end
+end

data/lib/bad_pigeon/timelines/user_timeline.rb

@@ -0,0 +1,26 @@
+require 'bad_pigeon/elements/timeline_instruction'
+require 'bad_pigeon/util/assertions'
+
+module BadPigeon
+  class UserTimeline
+    include Assertions
+
+    EXPECTED_INSTRUCTIONS = [
+      TimelineInstruction::Type::ADD_ENTRIES,
+      TimelineInstruction::Type::CLEAR_CACHE,
+      TimelineInstruction::Type::PIN_ENTRY
+    ]
+
+    def initialize(json)
+      @json = json
+    end
+
+    def instructions
+      @instructions ||= begin
+        list = @json['data']['user']['result']['timeline_v2']['timeline']['instructions']
+        assert { list.all? { |i| EXPECTED_INSTRUCTIONS.include?(i['type']) }}
+        list.map { |j| TimelineInstruction.new(j, self.class) }
+      end
+    end
+  end
+end

data/lib/bad_pigeon/timelines.rb

@@ -0,0 +1,10 @@
+Dir[File.join(__dir__, 'timelines', '*.rb')].each { |f| require(f) }
+
+module BadPigeon
+  TIMELINE_TYPES = {
+    'UserTweets' => UserTimeline,
+    'HomeLatestTimeline' => HomeTimeline,
+    'HomeTimeline' => HomeTimeline,
+    'ListLatestTweetsTimeline' => ListTimeline
+  }
+end

data/lib/bad_pigeon/tweet_extractor.rb

@@ -0,0 +1,37 @@
+require_relative 'har/har_archive'
+require_relative 'entry_filter'
+require_relative 'util/assertions'
+require_relative 'timelines'
+
+require 'uri'
+
+module BadPigeon
+  class TweetExtractor
+    include Assertions
+
+    def initialize
+      @filter = EntryFilter.new
+    end
+
+    def get_tweets_from_har(har_data)
+      archive = HARArchive.new(har_data)
+
+      requests = archive.entries.select { |e|
+        e.graphql_endpoint? && e.method == :get && e.status == 200 && e.has_json_response?
+      }
+
+      entries = requests.map { |e|
+        endpoint = URI(e.url).path.split('/').last
+
+        if timeline_class = TIMELINE_TYPES[endpoint]
+          timeline_class.new(e.response_json).instructions.map(&:entries)
+        elsif !TIMELINE_TYPES.has_key?(endpoint)
+          debug "Unknown endpoint: #{endpoint}"
+          []
+        end
+      }.flatten
+
+      entries.select { |e| @filter.include_entry?(e) }.map(&:items).flatten.map(&:tweet).compact
+    end
+  end
+end

data/lib/bad_pigeon/util/assertions.rb

@@ -0,0 +1,23 @@
+module AngryPigeon
+  class AssertionError < StandardError; end
+end
+
+module BadPigeon
+  module Assertions
+    def self.included(target)
+      if ENV['ANGRY_PIGEON'] == '1'
+        target.define_method(:assert) do |msg = nil, &block|
+          raise AngryPigeon::AssertionError.new(msg) if msg || block.call == false
+        end
+        target.define_method(:debug) do |msg|
+          $stderr.puts msg
+        end
+      else
+        target.define_method(:assert) do |msg = nil, &block|
+        end
+        target.define_method(:debug) do |msg|
+        end
+      end
+    end
+  end
+end

data/lib/bad_pigeon/util/strict_hash.rb

@@ -0,0 +1,16 @@
+require_relative 'assertions'
+
+module BadPigeon
+  class StrictHash < Hash
+    include Assertions
+
+    def [](key)
+      if has_key?(key)
+        super
+      else
+        assert("Missing hash key: #{key}")
+        nil
+      end
+    end
+  end
+end

data/lib/bad_pigeon/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BadPigeon
+  VERSION = "0.1.0"
+end

data/lib/bad_pigeon.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative 'bad_pigeon/tweet_extractor'
+require_relative 'bad_pigeon/version'
+
+module BadPigeon
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/sig/bad_pigeon.rbs

@@ -0,0 +1,4 @@
+module BadPigeon
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: bad_pigeon
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kuba Suder
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-06-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+description: "\n    BadPigeon is a Ruby gem that allows you to extract tweet data
+  from the XHR requests that the Twitter.com frontend\n    website does in user's
+  browser. The requests need to be saved into a \"HAR\" archive file from the browser's
+  web\n    inspector tool and then that file is fed into either the appropriate Ruby
+  class or the `pigeon` command line tool.\n    \n    The tool intents to be API compatible
+  with the popular `twitter` gem and generate the same kind of tweet JSON\n    structure
+  as is read and exported by that library.\n  "
+email:
+- jakub.suder@gmail.com
+executables:
+- pigeon
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- exe/pigeon
+- lib/bad_pigeon.rb
+- lib/bad_pigeon/elements/component.rb
+- lib/bad_pigeon/elements/timeline_entry.rb
+- lib/bad_pigeon/elements/timeline_instruction.rb
+- lib/bad_pigeon/elements/timeline_tweet.rb
+- lib/bad_pigeon/entry_filter.rb
+- lib/bad_pigeon/har/har_archive.rb
+- lib/bad_pigeon/har/har_request.rb
+- lib/bad_pigeon/models/tweet.rb
+- lib/bad_pigeon/models/url_entity.rb
+- lib/bad_pigeon/models/user.rb
+- lib/bad_pigeon/timelines.rb
+- lib/bad_pigeon/timelines/home_timeline.rb
+- lib/bad_pigeon/timelines/list_timeline.rb
+- lib/bad_pigeon/timelines/user_timeline.rb
+- lib/bad_pigeon/tweet_extractor.rb
+- lib/bad_pigeon/util/assertions.rb
+- lib/bad_pigeon/util/strict_hash.rb
+- lib/bad_pigeon/version.rb
+- sig/bad_pigeon.rbs
+homepage: https://github.com/mackuba/bad_pigeon
+licenses:
+- Zlib
+metadata:
+  bug_tracker_uri: https://github.com/mackuba/bad_pigeon/issues
+  changelog_uri: https://github.com/mackuba/bad_pigeon/blob/master/CHANGELOG.md
+  source_code_uri: https://github.com/mackuba/bad_pigeon
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: A tool for extracting tweet data from GraphQL fetch requests made by the
+  Twitter website
+test_files: []