tweet_sanitizer 0.1.0.beta1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 95ee13f344a020fee8f62754894bff6ce604aa41
-  data.tar.gz: 0dd22db74f75368271ed62f967141dd009290e40
+SHA256:
+  metadata.gz: 4c5714c30ff280753252d5c0a6bfe3365b329199edb2d670190d069e9ff7820b
+  data.tar.gz: 0157a784bc6b12391389d14a34a27f41b61805cefbbc272b2e3d1a1066e0fbe3
 SHA512:
-  metadata.gz: b9d8a7efbec32e451c484d4f2b9c0bc85d391d890c299a81cb7ac366a2773e6a3636b064ae21404e001331ae9eec156549cea0c459c63002f1f422edda3605f0
-  data.tar.gz: e075a6c9f8f5e6b29b5ae67e8294edb7f8a69a5f744d552969237a80c59487ef2092b1c87c9ebe85955d15776bfffd5e785fe3bac07a42a6acbd9637ddbb4bd1
+  metadata.gz: 18982482212aa7c854803a6770bb14b9d8990e9581fe7f3d27bc6896fd9645c6c408512d1b4e859d3c2c654095f7ce2ea7acb7e85686e542f49cdbdfb751f4d6
+  data.tar.gz: 6f82ab38dfdc1bcd8046d2cd1f4c8b28f239d5e5d4dd7f6d4a4af8c943d06a5a41e02d0ec2ab4129347dd610bb5cd203a60f9b9ef1c72bf851c022ea8757df48

data/.coveralls.yml

@@ -0,0 +1 @@
+repo_token: kZmA3FzhK4Oou7s7aQPRcAW0e8RbRlzhD

data/.github/workflows/test.yml

@@ -0,0 +1,111 @@
+name: test
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+  schedule:
+    - cron: "0 10 * * 5" # JST 19:00 (Fri)
+
+env:
+  CI: "true"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    container: ${{ matrix.ruby }}
+
+    strategy:
+      fail-fast: false
+
+      matrix:
+        ruby:
+          - ruby:2.2
+          - ruby:2.3
+          - ruby:2.4
+          - ruby:2.5
+          - ruby:2.6
+          - ruby:2.7
+          - ruby:3.0
+          - rubylang/ruby:master-nightly-bionic
+        include:
+          - ruby: rubylang/ruby:master-nightly-bionic
+            allow_failures: "true"
+
+    steps:
+      - uses: actions/checkout@v2
+
+
+      - name: Cache vendor/bundle
+        uses: actions/cache@v1
+        id: cache_gem
+        with:
+          path: vendor/bundle
+          key: v1-gem-${{ runner.os }}-${{ matrix.ruby }}-${{ github.sha }}
+          restore-keys: |
+            v1-gem-${{ runner.os }}-${{ matrix.ruby }}-
+        continue-on-error: ${{ matrix.allow_failures == 'true' }}
+
+      - name: bundle update
+        run: |
+          set -xe
+          bundle config path vendor/bundle
+          bundle update --jobs $(nproc) --retry 3
+        continue-on-error: ${{ matrix.allow_failures == 'true' }}
+
+      - name: Setup Code Climate Test Reporter
+        uses: aktions/codeclimate-test-reporter@v1
+        with:
+          codeclimate-test-reporter-id: ${{ secrets.CC_TEST_REPORTER_ID }}
+          command: before-build
+        if: matrix.ruby >= 'ruby:2.4'
+        continue-on-error: true
+
+      - name: Run test
+        run: |
+          set -xe
+          bundle exec rspec
+        continue-on-error: ${{ matrix.allow_failures == 'true' }}
+
+      - name: Teardown Code Climate Test Reporter
+        uses: aktions/codeclimate-test-reporter@v1
+        with:
+          codeclimate-test-reporter-id: ${{ secrets.CC_TEST_REPORTER_ID }}
+          command: after-build
+        if: matrix.ruby >= 'ruby:2.4' && always()
+        continue-on-error: true
+
+      - name: Slack Notification (not success)
+        uses: lazy-actions/slatify@master
+        if: "! success()"
+        continue-on-error: true
+        with:
+          job_name: ${{ format('*build* ({0})', matrix.ruby) }}
+          type: ${{ job.status }}
+          icon_emoji: ":octocat:"
+          url: ${{ secrets.SLACK_WEBHOOK }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  notify:
+    needs:
+      - test
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Slack Notification (success)
+        uses: lazy-actions/slatify@master
+        if: always()
+        continue-on-error: true
+        with:
+          job_name: '*build*'
+          type: ${{ job.status }}
+          icon_emoji: ":octocat:"
+          url: ${{ secrets.SLACK_WEBHOOK }}
+          token: ${{ secrets.GITHUB_TOKEN }}

data/CHANGELOG.md

@@ -0,0 +1,23 @@
+## Unreleased
+[full changelog](https://github.com/sue445/tweet_sanitizer/compare/v0.2.1...master)
+
+## v0.2.1
+[full changelog](https://github.com/sue445/tweet_sanitizer/compare/v0.2.0...v0.2.1)
+
+* Enable MFA requirement for gem releasing
+  * https://github.com/sue445/tweet_sanitizer/pull/32
+
+## v0.2.0
+[full changelog](https://github.com/sue445/tweet_sanitizer/compare/v0.1.1...v0.2.0)
+
+* Add sanitize options
+  * https://github.com/sue445/tweet_sanitizer/pull/12
+
+## v0.1.1
+[full changelog](https://github.com/sue445/tweet_sanitizer/compare/v0.1.0...v0.1.1)
+
+* Use original tweet instead of RT
+  * https://github.com/sue445/tweet_sanitizer/pull/10
+
+## v0.1.0
+* first release

data/Gemfile

@@ -4,3 +4,10 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in tweet_sanitizer.gemspec
 gemspec
+
+if Gem::Version.create(RUBY_VERSION) < Gem::Version.create("2.5.0")
+  group :test do
+    # NOTE: unparser v0.3.0+ requires Ruby 2.5+
+    gem "unparser", "< 0.3.0"
+  end
+end

data/README.md

@@ -1,8 +1,30 @@
 # TweetSanitizer
+[![Gem Version](https://badge.fury.io/rb/tweet_sanitizer.svg)](https://badge.fury.io/rb/tweet_sanitizer)
+[![Build Status](https://github.com/sue445/tweet_sanitizer/workflows/test/badge.svg?branch=master)](https://github.com/sue445/tweet_sanitizer/actions?query=workflow%3Atest)
+[![Code Climate](https://codeclimate.com/github/sue445/tweet_sanitizer.png)](https://codeclimate.com/github/sue445/tweet_sanitizer)
+[![Coverage Status](https://coveralls.io/repos/github/sue445/tweet_sanitizer/badge.svg?branch=master)](https://coveralls.io/github/sue445/tweet_sanitizer?branch=master)
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/tweet_sanitizer`. To experiment with that code, run `bin/console` for an interactive prompt.
+This is extension of [twitter](https://github.com/sferik/twitter). Sanitize `Twitter::Tweet#text`
 
-TODO: Delete this and the text above, and describe your gem
+## Example
+```ruby
+tweet = @client.status("https://twitter.com/github/status/866677968608927744")
+tweet.text
+#=> "Introducing GitHub Marketplace, a new place to browse and buy integrations using your GitHub account.… https://t.co/dK0Tmcmm72"
+```
+
+```ruby
+using TweetSanitizer::TwitterExtension
+
+extended_tweet = @client.status("https://twitter.com/github/status/866677968608927744", tweet_mode: "extended")
+extended_tweet.sanitized_text
+#=> "Introducing GitHub Marketplace, a new place to browse and buy integrations using your GitHub account. https://github.com/blog/2359-introducing-github-marketplace-and-more-tools-to-customize-your-workflow"
+```
+
+## Features
+* Expand urls in `Twitter::Tweet#text` (e.g. `t.co` url -> original url)
+* Remove media urls in `Twitter::Tweet#text`
+* Unescape special html characters in `Twitter::Tweet#text` (e.g. `(&gt; &lt;)` -> `(> <)`)
 
 ## Installation
 
@@ -21,8 +43,62 @@ Or install it yourself as:
     $ gem install tweet_sanitizer
 
 ## Usage
+### Case 1. Use `TweetSanitizer::TwitterExtension` and `Twitter::Tweet#sanitized_text` (Recommended)
+```ruby
+using TweetSanitizer::TwitterExtension
+
+extended_tweet = @client.status("https://twitter.com/github/status/866677968608927744", tweet_mode: "extended")
+extended_tweet.sanitized_text
+```
+
+### Case 2. Use `TweetSanitizer.sanitize`
+```ruby
+extended_tweet = @client.status("https://twitter.com/github/status/866677968608927744", tweet_mode: "extended")
+TweetSanitizer.sanitize(extended_tweet)
+```
+
+### :warning: Note
+When you get `Twitter::Tweet` instance, pass `tweet_mode: "extended"` option to methods. (e.g. `Twitter::REST::Client#status`, `Twitter::REST::Client#search`)
 
-TODO: Write usage instructions here
+Because `full_text` attribute is returned when **only** `tweet_mode=extended` is passed to Twitter API
+
+See following.
+
+https://dev.twitter.com/overview/api/upcoming-changes-to-tweets
+
+### Options
+Same to `Twitter::Tweet#sanitized_text` and `TweetSanitizer.sanitize`. (By default all are `true`)
+
+* `use_retweeted_tweet` : Whether use original retweeted tweet if exists
+* `expand_url` : Whether expand url in tweet (e.g. `t.co` url -> original url)
+* `remove_media_url` : Whether remove media url in tweet
+* `unescape` : Whether unescape in tweet (e.g. `(&gt; &lt;)` -> `(> <)`)
+
+#### Example
+
+```ruby
+tweet = @client.status("https://twitter.com/github/status/900456141557080065", tweet_mode: "extended")
+
+# all are false
+tweet.sanitized_text(use_retweeted_tweet: false, expand_url: false, remove_media_url: false, unescape: false)
+#=> "RT @GitHubEducation: Announcing two new GitHub Classroom features: Assignment Deadlines and Class Rosters: https://t.co/bNiJnlps5e https://…"
+
+# Only use_retweeted_tweet is true
+tweet.sanitized_text(use_retweeted_tweet: true, expand_url: false, remove_media_url: false, unescape: false)
+#=> "Announcing two new GitHub Classroom features: Assignment Deadlines and Class Rosters: https://t.co/bNiJnlps5e https://t.co/C02G05tUvu"
+
+# Only expand_url is true
+tweet.sanitized_text(use_retweeted_tweet: false, expand_url: true, remove_media_url: false, unescape: false)
+#=> "RT @GitHubEducation: Announcing two new GitHub Classroom features: Assignment Deadlines and Class Rosters: https://github.com/blog/2418-github-classroom-now-supports-deadlines-and-class-rosters https://…"
+
+# Only remove_media_url is true
+tweet.sanitized_text(use_retweeted_tweet: false, expand_url: false, remove_media_url: true, unescape: false)
+#=> "RT @GitHubEducation: Announcing two new GitHub Classroom features: Assignment Deadlines and Class Rosters: https://t.co/bNiJnlps5e https://…"
+
+# all are true (default)
+tweet.sanitized_text
+#=> "Announcing two new GitHub Classroom features: Assignment Deadlines and Class Rosters: https://github.com/blog/2418-github-classroom-now-supports-deadlines-and-class-rosters"
+```
 
 ## Development
 
@@ -33,13 +109,13 @@ vi .env
 ```
 
 ```ruby
-[1] pry(main)> @client.status("https://twitter.com/sue445/status/896748477262082049")
+[1] pry(main)> @client.status("https://twitter.com/sue445/status/896748477262082049", tweet_mode: "extended")
 => #<Twitter::Tweet id=896748477262082049>
 ```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/tweet_sanitizer.
+Bug reports and pull requests are welcome on GitHub at https://github.com/sue445/tweet_sanitizer.
 
 ## License
 

data/bin/console

@@ -15,4 +15,6 @@ Dotenv.load
   config.access_token_secret = ENV["TWITTER_ACCESS_TOKEN_SECRET"]
 end
 
+using TweetSanitizer::TwitterExtension
+
 Pry.start

data/lib/tweet_sanitizer/twitter_extension.rb

@@ -0,0 +1,21 @@
+module TweetSanitizer
+  module TwitterExtension
+    refine(Twitter::Tweet) do
+      # Sanitize tweet
+      # @param use_retweeted_tweet [Boolean] Whether use original retweeted tweet if exists
+      # @param expand_url          [Boolean] Whether expand url in tweet (e.g. `t.co` url -> original url)
+      # @param remove_media_url    [Boolean] Whether remove media url in tweet
+      # @param unescape            [Boolean] Whether unescape in tweet (e.g. `(&gt; &lt;)` -> `(> <)`)
+      # @return [String]
+      def sanitized_text(use_retweeted_tweet: true, expand_url: true, remove_media_url: true, unescape: true)
+        TweetSanitizer.sanitize(
+          self,
+          use_retweeted_tweet: use_retweeted_tweet,
+          expand_url:          expand_url,
+          remove_media_url:    remove_media_url,
+          unescape:            unescape,
+        )
+      end
+    end
+  end
+end

data/lib/tweet_sanitizer/version.rb

@@ -1,3 +1,3 @@
 module TweetSanitizer
-  VERSION = "0.1.0.beta1"
+  VERSION = "0.2.1"
 end

data/lib/tweet_sanitizer.rb

@@ -1,37 +1,56 @@
-require "tweet_sanitizer/version"
 require "twitter"
+require "tweet_sanitizer/version"
+require "tweet_sanitizer/twitter_extension"
 
 module TweetSanitizer
+  # Sanitize tweet
   # @param tweet [Twitter:Tweet]
+  # @param use_retweeted_tweet [Boolean] Whether use original retweeted tweet if exists
+  # @param expand_url          [Boolean] Whether expand url in tweet (e.g. `t.co` url -> original url)
+  # @param remove_media_url    [Boolean] Whether remove media url in tweet
+  # @param unescape            [Boolean] Whether unescape in tweet (e.g. `(&gt; &lt;)` -> `(> <)`)
   # @return [String]
-  def self.sanitize(tweet)
-    text = expand_urls_text(tweet)
-    text = remove_media_urls_in_tweet(tweet, text)
-    text = CGI.unescapeHTML(text)
+  def self.sanitize(tweet, use_retweeted_tweet: true, expand_url: true, remove_media_url: true, unescape: true)
+    # Original RT status is exists in retweeted_status
+    if use_retweeted_tweet && tweet.attrs[:retweeted_status]
+      tweet = Twitter::Tweet.new(tweet.attrs[:retweeted_status])
+    end
+
+    text = tweet_full_text(tweet)
+    text = expand_urls_text(tweet, text) if expand_url
+    text = remove_media_urls_in_tweet(tweet, text) if remove_media_url
+    text = CGI.unescapeHTML(text) if unescape
     text
   end
 
   # @param tweet [Twitter:Tweet]
+  # @param text [String]
   # @return [String]
-  def self.expand_urls_text(tweet)
-    text = tweet.text.dup
+  def self.expand_urls_text(tweet, text)
     return text unless tweet.uris?
 
-    tweet.uris.reverse.each_with_object(text) do |uri, expanded|
+    tweet.uris.reverse.each_with_object(text.dup) do |uri, expanded|
       pos1 = uri.indices[0]
       pos2 = uri.indices[1]
       expanded[pos1, pos2-pos1] = uri.expanded_url
     end
   end
 
+  # @param tweet [Twitter:Tweet]
+  # @return [String] `full_text` attribute if exist
+  def self.tweet_full_text(tweet)
+    # NOTE: Twitter::Tweet#full_text doesn't returns full_text attribute
+    # https://github.com/sferik/twitter/blob/v6.1.0/lib/twitter/tweet.rb#L37-L44
+    tweet.attrs[:full_text] || tweet.text
+  end
+
   # @param tweet [Twitter:Tweet]
   # @param text [String]
   # @return [String]
   def self.remove_media_urls_in_tweet(tweet, text)
-    text = text.dup
     return text unless tweet.media?
 
-    tweet.media.each_with_object(text) do |media, t|
+    tweet.media.each_with_object(text.dup) do |media, t|
       t.gsub!(media.url, "")
       t.strip!
     end

data/tweet_sanitizer.gemspec

@@ -14,6 +14,13 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/sue445/tweet_sanitizer"
   spec.license       = "MIT"
 
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.required_ruby_version = ">= 2.1.0"
+
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
@@ -23,10 +30,13 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "twitter"
 
-  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "bundler", ">= 1.15"
+  spec.add_development_dependency "coveralls"
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "pry-byebug"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-parameterized"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "unparser", ">= 0.4.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tweet_sanitizer
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta1
+  version: 0.2.1
 platform: ruby
 authors:
 - sue445
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-08-13 00:00:00.000000000 Z
+date: 2021-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -28,16 +28,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -70,14 +84,14 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
@@ -108,6 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unparser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.5
 description: sanitize tweet
 email:
 - sue445@sue445.net
@@ -115,10 +157,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".env.example"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -126,13 +170,18 @@ files:
 - bin/console
 - bin/setup
 - lib/tweet_sanitizer.rb
+- lib/tweet_sanitizer/twitter_extension.rb
 - lib/tweet_sanitizer/version.rb
 - tweet_sanitizer.gemspec
 homepage: https://github.com/sue445/tweet_sanitizer
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/sue445/tweet_sanitizer
+  source_code_uri: https://github.com/sue445/tweet_sanitizer
+  changelog_uri: https://github.com/sue445/tweet_sanitizer/blob/master/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -140,16 +189,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: sanitize tweet
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.4.1
-before_install: gem install bundler -v 1.15.3