x_post_sanitizer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c1b728cf95a2693f8bb1eb632e28ce05767f2c876854db333256e561ca50b323
+  data.tar.gz: 6b57921524d5c4f2891779184f432b8dbd215bf88452fb0eb37fa502bbb013c6
+SHA512:
+  metadata.gz: 02765f0f7e8e03d3185e4b8b56b7c07d5ace9830b0276b7fc2bc1b97d8cdf42124ad852d6bb5537a2a2b51f0b526570f9e17dcdb4567472c3fd9e55649ed2b90
+  data.tar.gz: 3ef6a6c9c5db8bab7571d7533e4bebd038b9966e67991b7e4d8bb8240126aba492c4a4435fe9520340dfdac7eb6e14073d2fc898d9c199ed7b89345d6783f16c

data/.rspec

@@ -0,0 +1,3 @@
+--format progress
+--color
+--require spec_helper

data/.yardopts

@@ -0,0 +1,6 @@
+--markup markdown
+--no-private
+--hide-void-return
+-
+CHANGELOG.md
+LICENSE.txt

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+## [Unreleased]
+[full changelog](http://github.com/sue445/x_post_sanitizer/compare/v0.1.0...main)
+
+## [0.1.0](https://github.com/sue445/x_post_sanitizer/releases/tag/v0.1.0) - 2025-05-04
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 sue445
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# XPostSanitizer
+Sanitize X Post (formerly Twitter Tweet)
+
+[![test](https://github.com/sue445/x_post_sanitizer/actions/workflows/test.yml/badge.svg)](https://github.com/sue445/x_post_sanitizer/actions/workflows/test.yml)
+
+## Installation
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add x_post_sanitizer
+```
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+```bash
+gem install x_post_sanitizer
+```
+
+## Usage
+Use `XPostSanitizer.sanitize_text`
+
+e.g
+
+```ruby
+require "json"
+require "x_post_sanitizer"
+
+# https://x.com/github/status/866677968608927744
+tweet = JSON.parse(File.read("spec/support/fixtures/full_text_tweet1.json"))
+
+tweet["full_text"]
+#=> "Introducing GitHub Marketplace, a new place to browse and buy integrations using your GitHub account. https://t.co/mPTtAxnU5z https://t.co/Wz2mUql2lc"
+
+XPostSanitizer.sanitize_text(tweet)
+#=> "Introducing GitHub Marketplace, a new place to browse and buy integrations using your GitHub account. https://github.com/blog/2359-introducing-github-marketplace-and-more-tools-to-customize-your-workflow"
+```
+
+## Features
+* Expand urls in `text` or `full_text` (e.g. `t.co` url -> original url)
+* Remove media urls in `text` or `full_text`
+* Unescape special html characters in `text` or `full_text` (e.g. `(&gt; &lt;)` -> `(> <)`)
+
+All methods and options are followings
+
+https://sue445.github.io/x_post_sanitizer/XPostSanitizer.html
+
+## vs tweet_sanitizer
+This has the same features as [tweet_sanitizer](https://github.com/sue445/tweet_sanitizer) but with the following differences
+
+* tweet_sanitizer
+  * Requires https://github.com/sferik/twitter-ruby
+* x_post_sanitizer
+  * Requires only plain X API response. (`Hash` which parsed JSON string)
+  * No requires https://github.com/sferik/twitter-ruby
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sue445/x_post_sanitizer.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+namespace :rbs do
+  desc "`rbs collection install` and `git commit`"
+  task :install do
+    sh "rbs collection install"
+    sh "git add rbs_collection.lock.yaml"
+    sh "git commit -m 'rbs collection install' || true"
+  end
+end
+
+desc "Check rbs"
+task :rbs do
+  sh "rbs validate"
+  sh "steep check"
+end
+
+task default: %i[spec rbs]

data/Steepfile

@@ -0,0 +1,34 @@
+# D = Steep::Diagnostic
+
+target :lib do
+  signature "sig"
+  ignore_signature "sig/test"
+
+  check "lib"                       # Directory name
+  check "path/to/source.rb"         # File name
+  check "app/models/**/*.rb"        # Glob
+  # ignore "lib/templates/*.rb"
+
+  # library "pathname"              # Standard libraries
+  # library "strong_json"           # Gems
+
+  collection_config "rbs_collection.yaml"
+
+  # configure_code_diagnostics(D::Ruby.default)      # `default` diagnostics setting (applies by default)
+  # configure_code_diagnostics(D::Ruby.strict)       # `strict` diagnostics setting
+  # configure_code_diagnostics(D::Ruby.lenient)      # `lenient` diagnostics setting
+  # configure_code_diagnostics(D::Ruby.silent)       # `silent` diagnostics setting
+  # configure_code_diagnostics do |hash|             # You can setup everything yourself
+  #   hash[D::Ruby::NoMethod] = :information
+  # end
+end
+
+# target :test do
+#   unreferenced!                     # Skip type checking the `lib` code when types in `test` target is changed
+#   signature "sig/test"              # Put RBS files for tests under `sig/test`
+#   check "test"                      # Type check Ruby scripts under `test`
+#
+#   configure_code_diagnostics(D::Ruby.lenient)      # Weak type checking for test code
+#
+#   # library "pathname"              # Standard libraries
+# end

data/lib/x_post_sanitizer/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module XPostSanitizer
+  VERSION = "0.1.0"
+end

data/lib/x_post_sanitizer.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative "x_post_sanitizer/version"
+
+module XPostSanitizer
+  class Error < StandardError; end
+
+  # Sanitize X Post (formerly Twitter Tweet)
+  #
+  # @param tweet [Hash<String, Object>] Tweet object
+  # @param use_retweeted_tweet [Boolean] Use original retweeted tweet if exists
+  # @param expand_url          [Boolean] Whether expand url in tweet (e.g. `t.co` url -> original url)
+  # @param remove_media_url    [Boolean] Whether remove media url in tweet
+  # @param unescape            [Boolean] Whether unescape in tweet (e.g. `(&gt; &lt;)` -> `(> <)`)
+  #
+  # @return [String] Sanitized text in tweet
+  #
+  # @see https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/tweet
+  def self.sanitize_text(tweet, use_retweeted_tweet: true, expand_url: true, remove_media_url: true, unescape: true)
+    # Original RT status exists in retweeted_status
+    tweet = tweet["retweeted_status"] if use_retweeted_tweet && tweet["retweeted_status"]
+
+    text = tweet_full_text(tweet)
+    text = expand_urls_in_text(tweet, text) if expand_url
+    text = remove_media_urls_in_tweet(tweet, text) if remove_media_url
+    text = CGI.unescapeHTML(text) if unescape
+    text
+  end
+
+  # @param tweet [Hash<String, Object>] Tweet object
+  # @param text [String]
+  #
+  # @return [String]
+  #
+  # @see https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/tweet
+  def self.expand_urls_in_text(tweet, text)
+    urls = tweet.dig("entities", "urls")
+
+    return text unless urls
+
+    urls.reverse.each_with_object(text.dup) do |url, expanded|
+      pos1 = url.dig("indices", 0)
+      pos2 = url.dig("indices", 1)
+      expanded[pos1, pos2-pos1] = url["expanded_url"] if url["expanded_url"] && pos1 && pos2
+    end
+  end
+
+  # @param tweet [Hash<String, Object>] Tweet object
+  #
+  # @return [String] `full_text` attribute if exist
+  #
+  # @see https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/tweet
+  def self.tweet_full_text(tweet)
+    tweet["full_text"] || tweet["text"]
+  end
+
+  # @param tweet [Hash<String, Object>] Tweet object
+  # @param text [String]
+  #
+  # @return [String]
+  #
+  # @see https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/tweet
+  def self.remove_media_urls_in_tweet(tweet, text)
+    medias = get_medias(tweet)
+
+    return text if medias.empty?
+
+    medias.each_with_object(text.dup) do |media, t|
+      t.gsub!(media["url"], "")
+      t.strip!
+    end
+  end
+
+  # @param tweet [Hash<String, Object>] Tweet object
+  #
+  # @return [Array<Hash>]
+  #
+  # @see https://developer.x.com/en/docs/x-api/v1/data-dictionary/object-model/tweet
+  def self.get_medias(tweet)
+    extended_entities_medias = tweet.dig("extended_entities", "media")
+    return extended_entities_medias if extended_entities_medias
+
+    tweet.dig("entities", "media") || []
+  end
+  private_class_method :get_medias
+end

data/rbs_collection.lock.yaml

@@ -0,0 +1,100 @@
+---
+path: ".gem_rbs_collection"
+gems:
+- name: ast
+  version: '2.4'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: eb6867fdb70e82e3aac6b51623ed1b6760c3c072
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: binding_of_caller
+  version: '1.0'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: eb6867fdb70e82e3aac6b51623ed1b6760c3c072
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: cgi
+  version: '0'
+  source:
+    type: stdlib
+- name: dbm
+  version: '0'
+  source:
+    type: stdlib
+- name: diff-lcs
+  version: '1.5'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: eb6867fdb70e82e3aac6b51623ed1b6760c3c072
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: fileutils
+  version: '0'
+  source:
+    type: stdlib
+- name: io-console
+  version: '0'
+  source:
+    type: stdlib
+- name: parser
+  version: '3.2'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: eb6867fdb70e82e3aac6b51623ed1b6760c3c072
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: pp
+  version: '0'
+  source:
+    type: stdlib
+- name: prettyprint
+  version: '0'
+  source:
+    type: stdlib
+- name: prism
+  version: 1.4.0
+  source:
+    type: rubygems
+- name: pstore
+  version: '0'
+  source:
+    type: stdlib
+- name: psych
+  version: '0'
+  source:
+    type: stdlib
+- name: rake
+  version: '13.0'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: eb6867fdb70e82e3aac6b51623ed1b6760c3c072
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: rdoc
+  version: '0'
+  source:
+    type: stdlib
+- name: rspec-parameterized-core
+  version: 2.0.0
+  source:
+    type: rubygems
+- name: rspec-parameterized-table_syntax
+  version: 2.0.0
+  source:
+    type: rubygems
+- name: stringio
+  version: '0'
+  source:
+    type: stdlib
+- name: tempfile
+  version: '0'
+  source:
+    type: stdlib
+gemfile_lock_path: Gemfile.lock

data/rbs_collection.yaml

@@ -0,0 +1,23 @@
+# Download sources
+sources:
+  - type: git
+    name: ruby/gem_rbs_collection
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    revision: main
+    repo_dir: gems
+
+# You can specify local directories as sources also.
+# - type: local
+#   path: path/to/your/local/repository
+
+# A directory to install the downloaded RBSs
+path: .gem_rbs_collection
+
+gems:
+  - name: cgi
+  - name: rbs
+    ignore: true
+  - name: steep
+    ignore: true
+  - name: yard
+    ignore: true

data/sig/x_post_sanitizer.rbs

@@ -0,0 +1,26 @@
+module XPostSanitizer
+  VERSION: String
+
+  class Error < StandardError
+  end
+
+  type tweet = Hash[String, untyped]
+
+  def self.sanitize_text: (
+    tweet tweet,
+    ?use_retweeted_tweet: bool,
+    ?expand_url: bool,
+    ?remove_media_url: bool,
+    ?unescape: bool
+  ) -> String
+
+  def self.expand_urls_in_text: (tweet tweet, String text) -> String
+
+  def self.tweet_full_text: (tweet tweet) -> String
+
+  def self.remove_media_urls_in_tweet: (tweet tweet, String text) -> String
+
+  def self.get_medias: (tweet tweet) -> Array[Hash[String, untyped]]
+
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: x_post_sanitizer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- sue445
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: irb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-parameterized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: steep
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Sanitize X Post (formerly Twitter Tweet)
+email:
+- sue445@sue445.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".yardopts"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- Steepfile
+- lib/x_post_sanitizer.rb
+- lib/x_post_sanitizer/version.rb
+- rbs_collection.lock.yaml
+- rbs_collection.yaml
+- sig/x_post_sanitizer.rbs
+homepage: https://github.com/sue445/x_post_sanitizer
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/sue445/x_post_sanitizer
+  source_code_uri: https://github.com/sue445/x_post_sanitizer
+  changelog_uri: https://github.com/sue445/x_post_sanitizer/blob/main/CHANGELOG.md
+  documentation_uri: https://sue445.github.io/x_post_sanitizer/
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Sanitize X Post (formerly Twitter Tweet)
+test_files: []