twitter-text 2.1.0 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +11 -1
- data/README.md +1 -1
- data/Rakefile +1 -1
- data/config/README.md +8 -0
- data/config/v3.json +30 -0
- data/lib/assets/tld_lib.yml +5 -0
- data/lib/twitter-text.rb +5 -0
- data/lib/twitter-text/autolink.rb +4 -0
- data/lib/twitter-text/configuration.rb +17 -3
- data/lib/twitter-text/deprecation.rb +4 -0
- data/lib/twitter-text/emoji_regex.rb +27 -0
- data/lib/twitter-text/extractor.rb +44 -16
- data/lib/twitter-text/hash_helper.rb +4 -0
- data/lib/twitter-text/hit_highlighter.rb +6 -2
- data/lib/twitter-text/regex.rb +28 -16
- data/lib/twitter-text/rewriter.rb +13 -9
- data/lib/twitter-text/unicode.rb +4 -0
- data/lib/twitter-text/validation.rb +39 -15
- data/lib/twitter-text/weighted_range.rb +4 -0
- data/spec/autolinking_spec.rb +4 -0
- data/spec/configuration_spec.rb +45 -0
- data/spec/extractor_spec.rb +4 -0
- data/spec/hithighlighter_spec.rb +4 -0
- data/spec/regex_spec.rb +38 -0
- data/spec/rewriter_spec.rb +4 -0
- data/spec/spec_helper.rb +13 -0
- data/spec/test_urls.rb +9 -5
- data/spec/twitter_text_spec.rb +4 -0
- data/spec/unicode_spec.rb +4 -0
- data/spec/validation_spec.rb +22 -2
- data/test/conformance_test.rb +20 -3
- data/twitter-text.gemspec +5 -2
- metadata +5 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 5ddb91693d013ffff5407a5ba5a6f08985221611
|
|
4
|
+
data.tar.gz: 85e60ba0ce0ad24a92d570204408f8cd78fd62df
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 259ad390fb78ea8e090ab17b2843b0c7a6b15ac2baf837278ae12449d36ea65cd37403cf7451b5c4288b8bb743469ae6da316d0150269b5eb6bf3d233d78dcfb
|
|
7
|
+
data.tar.gz: 5ad46e00e0d79a31bfac2a491dc690b93ea727743b900e665d76dfdc4f4b1f07e829b69525e5461b0760a73cfd3eefb28042bf880fe759f603e996a948179f76
|
data/CHANGELOG.md
CHANGED
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
All notable changes to this project will be documented in this file.
|
|
3
3
|
|
|
4
|
-
## [
|
|
4
|
+
## [3.0.0]
|
|
5
|
+
### Added
|
|
6
|
+
- New v3.json config file with emojiParsingEnabled config option. When
|
|
7
|
+
true, twitter-text will parse and discount emoji supported by the
|
|
8
|
+
twemoji library (see https://github.com/twitter/twemoji). The length
|
|
9
|
+
of these emoji will be the default weight (200 or two characters) even
|
|
10
|
+
if they contain multiple code points combined by zero-width
|
|
11
|
+
joiners. This means that emoji with skin tone and gender modifiers no
|
|
12
|
+
longer count as more characters than those without such modifiers.
|
|
13
|
+
### Changed
|
|
14
|
+
- Updates known gTLDs to recognize recent additions by IANA (#261)
|
|
5
15
|
|
|
6
16
|
## [2.1] - 2017-12-20
|
|
7
17
|
### Added
|
data/README.md
CHANGED
|
@@ -188,6 +188,6 @@ Have a bug? Please create an issue here on GitHub!
|
|
|
188
188
|
|
|
189
189
|
## License
|
|
190
190
|
|
|
191
|
-
Copyright 2012-
|
|
191
|
+
Copyright 2012-2018 Twitter, Inc and other contributors
|
|
192
192
|
|
|
193
193
|
Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
|
data/Rakefile
CHANGED
data/config/README.md
CHANGED
|
@@ -13,6 +13,7 @@ The configuration format is a JSON string. The JSON can have the following prope
|
|
|
13
13
|
* `maxWeightedTweetLength` (required, integer, min value 0)
|
|
14
14
|
* `scale` (required, integer, min value 1)
|
|
15
15
|
* `defaultWeight` (required, integer, min value 0)
|
|
16
|
+
* `emojiParsingEnabled` (optional, boolean)
|
|
16
17
|
* `transformedURLLength` (integer, min value 0)
|
|
17
18
|
* `ranges` (array of range items)
|
|
18
19
|
|
|
@@ -48,6 +49,13 @@ The Tweet length is the (`weighted length` / `scale`).
|
|
|
48
49
|
The default weight applied to all code points. This is overridden in
|
|
49
50
|
one or more range items.
|
|
50
51
|
|
|
52
|
+
### emojiParsingEnabled
|
|
53
|
+
|
|
54
|
+
When set to true, the weighted Tweet length considers all emoji as a
|
|
55
|
+
single code point (with a default weight of 200), including longer
|
|
56
|
+
grapheme clusters combined by zero-width joiners. When set to false,
|
|
57
|
+
Tweet length is calculated by weighing individual Unicode code points.
|
|
58
|
+
|
|
51
59
|
### transformedURLLength
|
|
52
60
|
|
|
53
61
|
The length counted for URLs against the total weight of the Tweet. In
|
data/config/v3.json
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"maxWeightedTweetLength": 280,
|
|
4
|
+
"scale": 100,
|
|
5
|
+
"defaultWeight": 200,
|
|
6
|
+
"emojiParsingEnabled": true,
|
|
7
|
+
"transformedURLLength": 23,
|
|
8
|
+
"ranges": [
|
|
9
|
+
{
|
|
10
|
+
"start": 0,
|
|
11
|
+
"end": 4351,
|
|
12
|
+
"weight": 100
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"start": 8192,
|
|
16
|
+
"end": 8205,
|
|
17
|
+
"weight": 100
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"start": 8208,
|
|
21
|
+
"end": 8223,
|
|
22
|
+
"weight": 100
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"start": 8242,
|
|
26
|
+
"end": 8247,
|
|
27
|
+
"weight": 100
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
data/lib/assets/tld_lib.yml
CHANGED
|
@@ -343,6 +343,7 @@ generic:
|
|
|
343
343
|
- 新闻
|
|
344
344
|
- 政府
|
|
345
345
|
- 政务
|
|
346
|
+
- 招聘
|
|
346
347
|
- 手表
|
|
347
348
|
- 手机
|
|
348
349
|
- 我爱你
|
|
@@ -598,6 +599,7 @@ generic:
|
|
|
598
599
|
- srl
|
|
599
600
|
- spreadbetting
|
|
600
601
|
- spot
|
|
602
|
+
- sport
|
|
601
603
|
- spiegel
|
|
602
604
|
- space
|
|
603
605
|
- soy
|
|
@@ -942,6 +944,7 @@ generic:
|
|
|
942
944
|
- locker
|
|
943
945
|
- loans
|
|
944
946
|
- loan
|
|
947
|
+
- llc
|
|
945
948
|
- lixil
|
|
946
949
|
- living
|
|
947
950
|
- live
|
|
@@ -1047,6 +1050,7 @@ generic:
|
|
|
1047
1050
|
- info
|
|
1048
1051
|
- infiniti
|
|
1049
1052
|
- industries
|
|
1053
|
+
- inc
|
|
1050
1054
|
- immobilien
|
|
1051
1055
|
- immo
|
|
1052
1056
|
- imdb
|
|
@@ -1364,6 +1368,7 @@ generic:
|
|
|
1364
1368
|
- cheap
|
|
1365
1369
|
- chat
|
|
1366
1370
|
- chase
|
|
1371
|
+
- charity
|
|
1367
1372
|
- channel
|
|
1368
1373
|
- chanel
|
|
1369
1374
|
- cfd
|
data/lib/twitter-text.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
major, minor, _patch = RUBY_VERSION.split('.')
|
|
2
6
|
|
|
3
7
|
$RUBY_1_9 = if major.to_i == 1 && minor.to_i < 9
|
|
@@ -10,6 +14,7 @@ end
|
|
|
10
14
|
|
|
11
15
|
%w(
|
|
12
16
|
deprecation
|
|
17
|
+
emoji_regex
|
|
13
18
|
regex
|
|
14
19
|
rewriter
|
|
15
20
|
autolink
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: UTF-8
|
|
2
6
|
|
|
3
7
|
module Twitter
|
|
@@ -6,7 +10,10 @@ module Twitter
|
|
|
6
10
|
require 'json'
|
|
7
11
|
|
|
8
12
|
PARSER_VERSION_CLASSIC = "v1"
|
|
9
|
-
|
|
13
|
+
PARSER_VERSION_WEIGHTED = "v2"
|
|
14
|
+
PARSER_VERSION_EMOJI_PARSING = "v3"
|
|
15
|
+
|
|
16
|
+
PARSER_VERSION_DEFAULT = PARSER_VERSION_WEIGHTED
|
|
10
17
|
|
|
11
18
|
class << self
|
|
12
19
|
attr_accessor :default_configuration
|
|
@@ -14,6 +21,7 @@ module Twitter
|
|
|
14
21
|
|
|
15
22
|
attr_reader :version, :max_weighted_tweet_length, :scale
|
|
16
23
|
attr_reader :default_weight, :transformed_url_length, :ranges
|
|
24
|
+
attr_reader :emoji_parsing_enabled
|
|
17
25
|
|
|
18
26
|
CONFIG_V1 = File.join(
|
|
19
27
|
File.expand_path('../../../config', __FILE__), # project root
|
|
@@ -22,7 +30,12 @@ module Twitter
|
|
|
22
30
|
|
|
23
31
|
CONFIG_V2 = File.join(
|
|
24
32
|
File.expand_path('../../../config', __FILE__), # project root
|
|
25
|
-
"#{
|
|
33
|
+
"#{PARSER_VERSION_WEIGHTED}.json"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
CONFIG_V3 = File.join(
|
|
37
|
+
File.expand_path('../../../config', __FILE__), # project root
|
|
38
|
+
"#{PARSER_VERSION_EMOJI_PARSING}.json"
|
|
26
39
|
)
|
|
27
40
|
|
|
28
41
|
def self.parse_string(string, options = {})
|
|
@@ -45,10 +58,11 @@ module Twitter
|
|
|
45
58
|
@scale = config[:scale]
|
|
46
59
|
@default_weight = config[:defaultWeight]
|
|
47
60
|
@transformed_url_length = config[:transformedURLLength]
|
|
61
|
+
@emoji_parsing_enabled = config[:emojiParsingEnabled]
|
|
48
62
|
@ranges = config[:ranges].map { |range| Twitter::TwitterText::WeightedRange.new(range) } if config.key?(:ranges) && config[:ranges].is_a?(Array)
|
|
49
63
|
end
|
|
50
64
|
|
|
51
|
-
self.default_configuration = self.configuration_from_file(
|
|
65
|
+
self.default_configuration = self.configuration_from_file(CONFIG_V3)
|
|
52
66
|
end
|
|
53
67
|
end
|
|
54
68
|
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
#
|
|
2
|
+
# emoji_regex.rb
|
|
3
|
+
#
|
|
4
|
+
# Copyright © 2018 Twitter. All rights reserved.
|
|
5
|
+
#
|
|
6
|
+
# DO NOT MODIFY THIS FILE -- it is generated for twitter-text automatically
|
|
7
|
+
|
|
8
|
+
# encoding: utf-8
|
|
9
|
+
|
|
10
|
+
module Twitter
|
|
11
|
+
module TwitterText
|
|
12
|
+
class Regex
|
|
13
|
+
class Emoji
|
|
14
|
+
REGEXEN = {} # :nodoc:
|
|
15
|
+
|
|
16
|
+
# This regex pattern matches a single emoji
|
|
17
|
+
REGEXEN[:valid_emoji] = %r{
|
|
18
|
+
[\u{01f468}\u{01f469}][\u{01f3fb}-\u{01f3ff}]?\u200d(?:\u2695\ufe0f|\u2696\ufe0f|\u2708\ufe0f|[\u{01f33e}\u{01f373}\u{01f393}\u{01f3a4}\u{01f3a8}\u{01f3eb}\u{01f3ed}\u{01f4bb}\u{01f4bc}\u{01f527}\u{01f52c}\u{01f680}\u{01f692}\u{01f9b0}-\u{01f9b3}])|[\u26f9\u{01f3cb}\u{01f3cc}\u{01f574}\u{01f575}](?:[\ufe0f\u{01f3fb}-\u{01f3ff}]\u200d[\u2640\u2642]\ufe0f)|[\u{01f3c3}\u{01f3c4}\u{01f3ca}\u{01f46e}\u{01f471}\u{01f473}\u{01f477}\u{01f481}\u{01f482}\u{01f486}\u{01f487}\u{01f645}-\u{01f647}\u{01f64b}\u{01f64d}\u{01f64e}\u{01f6a3}\u{01f6b4}-\u{01f6b6}\u{01f926}\u{01f935}\u{01f937}-\u{01f939}\u{01f93d}\u{01f93e}\u{01f9b8}\u{01f9b9}\u{01f9d6}-\u{01f9dd}][\u{01f3fb}-\u{01f3ff}]?\u200d[\u2640\u2642]\ufe0f|(?:\u{01f468}\u200d\u2764\ufe0f\u200d\u{01f48b}\u200d\u{01f468}|\u{01f469}\u200d\u2764\ufe0f\u200d\u{01f48b}\u200d[\u{01f468}\u{01f469}]|\u{01f468}\u200d\u{01f468}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f468}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f469}\u200d\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u2764\ufe0f\u200d\u{01f468}|\u{01f469}\u200d\u2764\ufe0f\u200d[\u{01f468}\u{01f469}]|\u{01f468}\u200d\u{01f466}\u200d\u{01f466}|\u{01f468}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f468}\u200d[\u{01f466}\u{01f467}]|\u{01f468}\u200d\u{01f469}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f466}\u200d\u{01f466}|\u{01f469}\u200d\u{01f467}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d\u{01f469}\u200d[\u{01f466}\u{01f467}]|\u{01f3f3}\ufe0f\u200d\u{01f308}|\u{01f3f4}\u200d\u2620\ufe0f|\u{01f46f}\u200d\u2640\ufe0f|\u{01f46f}\u200d\u2642\ufe0f|\u{01f93c}\u200d\u2640\ufe0f|\u{01f93c}\u200d\u2642\ufe0f|\u{01f9de}\u200d\u2640\ufe0f|\u{01f9de}\u200d\u2642\ufe0f|\u{01f9df}\u200d\u2640\ufe0f|\u{01f9df}\u200d\u2642\ufe0f|\u{01f441}\u200d\u{01f5e8}|\u{01f468}\u200d[\u{01f466}\u{01f467}]|\u{01f469}\u200d[\u{01f466}\u{01f467}])|[#*0-9]\ufe0f?\u20e3|(?:[©®\u2122\u265f]\ufe0f)|[\u203c\u2049\u2139\u2194-\u2199\u21a9\u21aa\u231a\u231b\u2328\u23cf\u23ed-\u23ef\u23f1\u23f2\u23f8-\u23fa\u24c2\u25aa\u25ab\u25b6\u25c0\u25fb-\u25fe\u2600-\u2604\u260e\u2611\u2614\u2615\u2618\u2620\u2622\u2623\u2626\u262a\u262e\u262f\u2638-\u263a\u2640\u2642\u2648-\u2653\u2660\u2663\u2665\u2666\u2668\u267b\u267f\u2692-\u2697\u2699\u269b\u269c\u26a0\u26a1\u26aa\u26ab\u26b0\u26b1\u26bd\u26be\u26c4\u26c5\u26c8\u26cf\u26d1\u26d3\u26d4\u26e9\u26ea\u26f0-\u26f5\u26f8\u26fa\u26fd\u2702\u2708\u2709\u270f\u2712\u2714\u2716\u271d\u2721\u2733\u2734\u2744\u2747\u2757\u2763\u2764\u27a1\u2934\u2935\u2b05-\u2b07\u2b1b\u2b1c\u2b50\u2b55\u3030\u303d\u3297\u3299\u{01f004}\u{01f170}\u{01f171}\u{01f17e}\u{01f17f}\u{01f202}\u{01f21a}\u{01f22f}\u{01f237}\u{01f321}\u{01f324}-\u{01f32c}\u{01f336}\u{01f37d}\u{01f396}\u{01f397}\u{01f399}-\u{01f39b}\u{01f39e}\u{01f39f}\u{01f3cd}\u{01f3ce}\u{01f3d4}-\u{01f3df}\u{01f3f3}\u{01f3f5}\u{01f3f7}\u{01f43f}\u{01f441}\u{01f4fd}\u{01f549}\u{01f54a}\u{01f56f}\u{01f570}\u{01f573}\u{01f576}-\u{01f579}\u{01f587}\u{01f58a}-\u{01f58d}\u{01f5a5}\u{01f5a8}\u{01f5b1}\u{01f5b2}\u{01f5bc}\u{01f5c2}-\u{01f5c4}\u{01f5d1}-\u{01f5d3}\u{01f5dc}-\u{01f5de}\u{01f5e1}\u{01f5e3}\u{01f5e8}\u{01f5ef}\u{01f5f3}\u{01f5fa}\u{01f6cb}\u{01f6cd}-\u{01f6cf}\u{01f6e0}-\u{01f6e5}\u{01f6e9}\u{01f6f0}\u{01f6f3}](?:\ufe0f|(?!\ufe0e))|(?:[\u261d\u26f7\u26f9\u270c\u270d\u{01f3cb}\u{01f3cc}\u{01f574}\u{01f575}\u{01f590}](?:\ufe0f|(?!\ufe0e))|[\u270a\u270b\u{01f385}\u{01f3c2}-\u{01f3c4}\u{01f3c7}\u{01f3ca}\u{01f442}\u{01f443}\u{01f446}-\u{01f450}\u{01f466}-\u{01f469}\u{01f46e}\u{01f470}-\u{01f478}\u{01f47c}\u{01f481}-\u{01f483}\u{01f485}-\u{01f487}\u{01f4aa}\u{01f57a}\u{01f595}\u{01f596}\u{01f645}-\u{01f647}\u{01f64b}-\u{01f64f}\u{01f6a3}\u{01f6b4}-\u{01f6b6}\u{01f6c0}\u{01f6cc}\u{01f918}-\u{01f91c}\u{01f91e}\u{01f91f}\u{01f926}\u{01f930}-\u{01f939}\u{01f93d}\u{01f93e}\u{01f9b5}\u{01f9b6}\u{01f9b8}\u{01f9b9}\u{01f9d1}-\u{01f9dd}])[\u{01f3fb}-\u{01f3ff}]?|(?:\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0065}\u{0e006e}\u{0e0067}\u{0e007f}|\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0073}\u{0e0063}\u{0e0074}\u{0e007f}|\u{01f3f4}\u{0e0067}\u{0e0062}\u{0e0077}\u{0e006c}\u{0e0073}\u{0e007f}|\u{01f1e6}[\u{01f1e8}-\u{01f1ec}\u{01f1ee}\u{01f1f1}\u{01f1f2}\u{01f1f4}\u{01f1f6}-\u{01f1fa}\u{01f1fc}\u{01f1fd}\u{01f1ff}]|\u{01f1e7}[\u{01f1e6}\u{01f1e7}\u{01f1e9}-\u{01f1ef}\u{01f1f1}-\u{01f1f4}\u{01f1f6}-\u{01f1f9}\u{01f1fb}\u{01f1fc}\u{01f1fe}\u{01f1ff}]|\u{01f1e8}[\u{01f1e6}\u{01f1e8}\u{01f1e9}\u{01f1eb}-\u{01f1ee}\u{01f1f0}-\u{01f1f5}\u{01f1f7}\u{01f1fa}-\u{01f1ff}]|\u{01f1e9}[\u{01f1ea}\u{01f1ec}\u{01f1ef}\u{01f1f0}\u{01f1f2}\u{01f1f4}\u{01f1ff}]|\u{01f1ea}[\u{01f1e6}\u{01f1e8}\u{01f1ea}\u{01f1ec}\u{01f1ed}\u{01f1f7}-\u{01f1fa}]|\u{01f1eb}[\u{01f1ee}-\u{01f1f0}\u{01f1f2}\u{01f1f4}\u{01f1f7}]|\u{01f1ec}[\u{01f1e6}\u{01f1e7}\u{01f1e9}-\u{01f1ee}\u{01f1f1}-\u{01f1f3}\u{01f1f5}-\u{01f1fa}\u{01f1fc}\u{01f1fe}]|\u{01f1ed}[\u{01f1f0}\u{01f1f2}\u{01f1f3}\u{01f1f7}\u{01f1f9}\u{01f1fa}]|\u{01f1ee}[\u{01f1e8}-\u{01f1ea}\u{01f1f1}-\u{01f1f4}\u{01f1f6}-\u{01f1f9}]|\u{01f1ef}[\u{01f1ea}\u{01f1f2}\u{01f1f4}\u{01f1f5}]|\u{01f1f0}[\u{01f1ea}\u{01f1ec}-\u{01f1ee}\u{01f1f2}\u{01f1f3}\u{01f1f5}\u{01f1f7}\u{01f1fc}\u{01f1fe}\u{01f1ff}]|\u{01f1f1}[\u{01f1e6}-\u{01f1e8}\u{01f1ee}\u{01f1f0}\u{01f1f7}-\u{01f1fb}\u{01f1fe}]|\u{01f1f2}[\u{01f1e6}\u{01f1e8}-\u{01f1ed}\u{01f1f0}-\u{01f1ff}]|\u{01f1f3}[\u{01f1e6}\u{01f1e8}\u{01f1ea}-\u{01f1ec}\u{01f1ee}\u{01f1f1}\u{01f1f4}\u{01f1f5}\u{01f1f7}\u{01f1fa}\u{01f1ff}]|\u{01f1f4}\u{01f1f2}|\u{01f1f5}[\u{01f1e6}\u{01f1ea}-\u{01f1ed}\u{01f1f0}-\u{01f1f3}\u{01f1f7}-\u{01f1f9}\u{01f1fc}\u{01f1fe}]|\u{01f1f6}\u{01f1e6}|\u{01f1f7}[\u{01f1ea}\u{01f1f4}\u{01f1f8}\u{01f1fa}\u{01f1fc}]|\u{01f1f8}[\u{01f1e6}-\u{01f1ea}\u{01f1ec}-\u{01f1f4}\u{01f1f7}-\u{01f1f9}\u{01f1fb}\u{01f1fd}-\u{01f1ff}]|\u{01f1f9}[\u{01f1e6}\u{01f1e8}\u{01f1e9}\u{01f1eb}-\u{01f1ed}\u{01f1ef}-\u{01f1f4}\u{01f1f7}\u{01f1f9}\u{01f1fb}\u{01f1fc}\u{01f1ff}]|\u{01f1fa}[\u{01f1e6}\u{01f1ec}\u{01f1f2}\u{01f1f3}\u{01f1f8}\u{01f1fe}\u{01f1ff}]|\u{01f1fb}[\u{01f1e6}\u{01f1e8}\u{01f1ea}\u{01f1ec}\u{01f1ee}\u{01f1f3}\u{01f1fa}]|\u{01f1fc}[\u{01f1eb}\u{01f1f8}]|\u{01f1fd}\u{01f1f0}|\u{01f1fe}[\u{01f1ea}\u{01f1f9}]|\u{01f1ff}[\u{01f1e6}\u{01f1f2}\u{01f1fc}]|[\u23e9-\u23ec\u23f0\u23f3\u267e\u26ce\u2705\u2728\u274c\u274e\u2753-\u2755\u2795-\u2797\u27b0\u27bf\ue50a\u{01f0cf}\u{01f18e}\u{01f191}-\u{01f19a}\u{01f1e6}-\u{01f1ff}\u{01f201}\u{01f232}-\u{01f236}\u{01f238}-\u{01f23a}\u{01f250}\u{01f251}\u{01f300}-\u{01f320}\u{01f32d}-\u{01f335}\u{01f337}-\u{01f37c}\u{01f37e}-\u{01f384}\u{01f386}-\u{01f393}\u{01f3a0}-\u{01f3c1}\u{01f3c5}\u{01f3c6}\u{01f3c8}\u{01f3c9}\u{01f3cf}-\u{01f3d3}\u{01f3e0}-\u{01f3f0}\u{01f3f4}\u{01f3f8}-\u{01f43e}\u{01f440}\u{01f444}\u{01f445}\u{01f451}-\u{01f465}\u{01f46a}-\u{01f46d}\u{01f46f}\u{01f479}-\u{01f47b}\u{01f47d}-\u{01f480}\u{01f484}\u{01f488}-\u{01f4a9}\u{01f4ab}-\u{01f4fc}\u{01f4ff}-\u{01f53d}\u{01f54b}-\u{01f54e}\u{01f550}-\u{01f567}\u{01f5a4}\u{01f5fb}-\u{01f644}\u{01f648}-\u{01f64a}\u{01f680}-\u{01f6a2}\u{01f6a4}-\u{01f6b3}\u{01f6b7}-\u{01f6bf}\u{01f6c1}-\u{01f6c5}\u{01f6d0}-\u{01f6d2}\u{01f6eb}\u{01f6ec}\u{01f6f4}-\u{01f6f9}\u{01f910}-\u{01f917}\u{01f91d}\u{01f920}-\u{01f925}\u{01f927}-\u{01f92f}\u{01f93a}\u{01f93c}\u{01f940}-\u{01f945}\u{01f947}-\u{01f970}\u{01f973}-\u{01f976}\u{01f97a}\u{01f97c}-\u{01f9a2}\u{01f9b4}\u{01f9b7}\u{01f9c0}-\u{01f9c2}\u{01f9d0}\u{01f9de}-\u{01f9ff}])|\ufe0f
|
|
19
|
+
}iox
|
|
20
|
+
|
|
21
|
+
def self.[](key)
|
|
22
|
+
REGEXEN[key]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
require 'idn'
|
|
3
7
|
|
|
@@ -5,7 +9,7 @@ class String
|
|
|
5
9
|
# Helper function to count the character length by first converting to an
|
|
6
10
|
# array. This is needed because with unicode strings, the return value
|
|
7
11
|
# of length may be incorrect
|
|
8
|
-
def
|
|
12
|
+
def codepoint_length
|
|
9
13
|
if respond_to? :codepoints
|
|
10
14
|
length
|
|
11
15
|
else
|
|
@@ -13,25 +17,25 @@ class String
|
|
|
13
17
|
end
|
|
14
18
|
end
|
|
15
19
|
|
|
16
|
-
# Helper function to convert this string into an array of unicode
|
|
17
|
-
def
|
|
18
|
-
@
|
|
20
|
+
# Helper function to convert this string into an array of unicode code points.
|
|
21
|
+
def to_codepoint_a
|
|
22
|
+
@to_codepoint_a ||= if chars.kind_of?(Enumerable)
|
|
19
23
|
chars.to_a
|
|
20
24
|
else
|
|
21
|
-
|
|
22
|
-
0.upto(
|
|
23
|
-
|
|
25
|
+
codepoint_array = []
|
|
26
|
+
0.upto(codepoint_length - 1) { |i| codepoint_array << [chars.slice(i)].pack('U') }
|
|
27
|
+
codepoint_array
|
|
24
28
|
end
|
|
25
29
|
end
|
|
26
30
|
end
|
|
27
31
|
|
|
28
|
-
# Helper functions to return
|
|
32
|
+
# Helper functions to return code point offsets instead of byte offsets.
|
|
29
33
|
class MatchData
|
|
30
34
|
def char_begin(n)
|
|
31
35
|
if string.respond_to? :codepoints
|
|
32
36
|
self.begin(n)
|
|
33
37
|
else
|
|
34
|
-
string[0, self.begin(n)].
|
|
38
|
+
string[0, self.begin(n)].codepoint_length
|
|
35
39
|
end
|
|
36
40
|
end
|
|
37
41
|
|
|
@@ -39,7 +43,7 @@ class MatchData
|
|
|
39
43
|
if string.respond_to? :codepoints
|
|
40
44
|
self.end(n)
|
|
41
45
|
else
|
|
42
|
-
string[0, self.end(n)].
|
|
46
|
+
string[0, self.end(n)].codepoint_length
|
|
43
47
|
end
|
|
44
48
|
end
|
|
45
49
|
end
|
|
@@ -77,11 +81,14 @@ module Twitter
|
|
|
77
81
|
#
|
|
78
82
|
# If a block is given then it will be called for each entity.
|
|
79
83
|
def extract_entities_with_indices(text, options = {}, &block)
|
|
84
|
+
config = options[:config] || Twitter::TwitterText::Configuration.default_configuration
|
|
85
|
+
|
|
80
86
|
# extract all entities
|
|
81
87
|
entities = extract_urls_with_indices(text, options) +
|
|
82
88
|
extract_hashtags_with_indices(text, :check_url_overlap => false) +
|
|
83
89
|
extract_mentions_or_lists_with_indices(text) +
|
|
84
90
|
extract_cashtags_with_indices(text)
|
|
91
|
+
entities += extract_emoji_with_indices(text) if config.emoji_parsing_enabled
|
|
85
92
|
|
|
86
93
|
return [] if entities.empty?
|
|
87
94
|
|
|
@@ -218,11 +225,7 @@ module Twitter
|
|
|
218
225
|
:indices => [start_position + $~.char_begin(0),
|
|
219
226
|
start_position + $~.char_end(0)]
|
|
220
227
|
}
|
|
221
|
-
|
|
222
|
-
ascii_domain =~ Twitter::TwitterText::Regex[:valid_special_short_domain] ||
|
|
223
|
-
ascii_domain !~ Twitter::TwitterText::Regex[:invalid_short_domain]
|
|
224
|
-
urls << last_url
|
|
225
|
-
end
|
|
228
|
+
urls << last_url
|
|
226
229
|
end
|
|
227
230
|
|
|
228
231
|
# no ASCII-only domain found. Skip the entire URL
|
|
@@ -239,7 +242,7 @@ module Twitter
|
|
|
239
242
|
if url =~ Twitter::TwitterText::Regex[:valid_tco_url]
|
|
240
243
|
next if $1 && $1.length > MAX_TCO_SLUG_LENGTH
|
|
241
244
|
url = $&
|
|
242
|
-
end_position = start_position + url.
|
|
245
|
+
end_position = start_position + url.codepoint_length
|
|
243
246
|
end
|
|
244
247
|
|
|
245
248
|
next unless is_valid_domain(url.length, domain, protocol)
|
|
@@ -341,6 +344,31 @@ module Twitter
|
|
|
341
344
|
tags
|
|
342
345
|
end
|
|
343
346
|
|
|
347
|
+
def extract_emoji_with_indices(text) # :yields: emoji, start, end
|
|
348
|
+
emoji = []
|
|
349
|
+
text.scan(Twitter::TwitterText::Regex[:valid_emoji]) do |emoji_text|
|
|
350
|
+
match_data = $~
|
|
351
|
+
start_position = match_data.char_begin(0)
|
|
352
|
+
end_position = match_data.char_end(0)
|
|
353
|
+
emoji << {
|
|
354
|
+
:emoji => emoji_text,
|
|
355
|
+
:indices => [start_position, end_position]
|
|
356
|
+
}
|
|
357
|
+
end
|
|
358
|
+
emoji
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def is_valid_emoji(text)
|
|
362
|
+
begin
|
|
363
|
+
raise ArgumentError.new("invalid empty emoji") unless text
|
|
364
|
+
entities = extract_emoji_with_indices(text)
|
|
365
|
+
entities.count == 1 && entities[0][:emoji] == text
|
|
366
|
+
rescue Exception
|
|
367
|
+
# On error don't consider this a valid domain.
|
|
368
|
+
return false
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
|
|
344
372
|
def is_valid_domain(url_length, domain, protocol)
|
|
345
373
|
begin
|
|
346
374
|
raise ArgumentError.new("invalid empty domain") unless domain
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
module Twitter
|
|
2
6
|
module TwitterText
|
|
3
7
|
# Module for doing "hit highlighting" on tweets that have been auto-linked already.
|
|
@@ -26,7 +30,7 @@ module Twitter
|
|
|
26
30
|
|
|
27
31
|
result = []
|
|
28
32
|
chunk_index, chunk = 0, chunks[0]
|
|
29
|
-
chunk_chars = chunk.to_s.
|
|
33
|
+
chunk_chars = chunk.to_s.to_codepoint_a
|
|
30
34
|
prev_chunks_len = 0
|
|
31
35
|
chunk_cursor = 0
|
|
32
36
|
start_in_chunk = false
|
|
@@ -50,7 +54,7 @@ module Twitter
|
|
|
50
54
|
chunk_cursor = 0
|
|
51
55
|
chunk_index += 2
|
|
52
56
|
chunk = chunks[chunk_index]
|
|
53
|
-
chunk_chars = chunk.to_s.
|
|
57
|
+
chunk_chars = chunk.to_s.to_codepoint_a
|
|
54
58
|
start_in_chunk = false
|
|
55
59
|
end
|
|
56
60
|
|
data/lib/twitter-text/regex.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
|
|
3
7
|
module Twitter
|
|
@@ -53,11 +57,26 @@ module Twitter
|
|
|
53
57
|
].flatten.map{|c| [c].pack('U*')}.freeze
|
|
54
58
|
REGEXEN[:spaces] = /[#{UNICODE_SPACES.join('')}]/o
|
|
55
59
|
|
|
60
|
+
DIRECTIONAL_CHARACTERS = [
|
|
61
|
+
0x061C, # ARABIC LETTER MARK (ALM)
|
|
62
|
+
0x200E, # LEFT-TO-RIGHT MARK (LRM)
|
|
63
|
+
0x200F, # RIGHT-TO-LEFT MARK (RLM)
|
|
64
|
+
0x202A, # LEFT-TO-RIGHT EMBEDDING (LRE)
|
|
65
|
+
0x202B, # RIGHT-TO-LEFT EMBEDDING (RLE)
|
|
66
|
+
0x202C, # POP DIRECTIONAL FORMATTING (PDF)
|
|
67
|
+
0x202D, # LEFT-TO-RIGHT OVERRIDE (LRO)
|
|
68
|
+
0x202E, # RIGHT-TO-LEFT OVERRIDE (RLO)
|
|
69
|
+
0x2066, # LEFT-TO-RIGHT ISOLATE (LRI)
|
|
70
|
+
0x2067, # RIGHT-TO-LEFT ISOLATE (RLI)
|
|
71
|
+
0x2068, # FIRST STRONG ISOLATE (FSI)
|
|
72
|
+
0x2069, # POP DIRECTIONAL ISOLATE (PDI)
|
|
73
|
+
].map{|cp| [cp].pack('U')}.freeze
|
|
74
|
+
REGEXEN[:directional_characters] = /[#{DIRECTIONAL_CHARACTERS.join('')}]/o
|
|
75
|
+
|
|
56
76
|
# Character not allowed in Tweets
|
|
57
77
|
INVALID_CHARACTERS = [
|
|
58
78
|
0xFFFE, 0xFEFF, # BOM
|
|
59
79
|
0xFFFF, # Special
|
|
60
|
-
0x202A, 0x202B, 0x202C, 0x202D, 0x202E # Directional change
|
|
61
80
|
].map{|cp| [cp].pack('U') }.freeze
|
|
62
81
|
REGEXEN[:invalid_control_characters] = /[#{INVALID_CHARACTERS.join('')}]/o
|
|
63
82
|
|
|
@@ -157,14 +176,16 @@ module Twitter
|
|
|
157
176
|
([a-z0-9_]{1,20}) # $3: Screen name
|
|
158
177
|
(\/[a-z][a-zA-Z0-9_\-]{0,24})? # $4: List (optional)
|
|
159
178
|
/iox
|
|
160
|
-
REGEXEN[:valid_reply] = /^(
|
|
179
|
+
REGEXEN[:valid_reply] = /^(?:[#{UNICODE_SPACES}#{DIRECTIONAL_CHARACTERS}])*#{REGEXEN[:at_signs]}([a-z0-9_]{1,20})/io
|
|
161
180
|
# Used in Extractor for final filtering
|
|
162
181
|
REGEXEN[:end_mention_match] = /\A(?:#{REGEXEN[:at_signs]}|#{REGEXEN[:latin_accents]}|:\/\/)/io
|
|
163
182
|
|
|
164
183
|
# URL related hash regex collection
|
|
165
|
-
REGEXEN[:valid_url_preceding_chars] = /(?:[^A-Z0-9@＠$#＃#{INVALID_CHARACTERS.join('')}]|^)/io
|
|
184
|
+
REGEXEN[:valid_url_preceding_chars] = /(?:[^A-Z0-9@＠$#＃#{INVALID_CHARACTERS.join('')}]|[#{DIRECTIONAL_CHARACTERS.join('')}]|^)/io
|
|
166
185
|
REGEXEN[:invalid_url_without_protocol_preceding_chars] = /[-_.\/]$/
|
|
167
|
-
|
|
186
|
+
|
|
187
|
+
DOMAIN_VALID_CHARS = "[^#{DIRECTIONAL_CHARACTERS.join('')}#{PUNCTUATION_CHARS}#{SPACE_CHARS}#{CTRL_CHARS}#{INVALID_CHARACTERS.join('')}#{UNICODE_SPACES.join('')}]"
|
|
188
|
+
# "[a-z0-9#{LATIN_ACCENTS}]"
|
|
168
189
|
REGEXEN[:valid_subdomain] = /(?:(?:#{DOMAIN_VALID_CHARS}(?:[_-]|#{DOMAIN_VALID_CHARS})*)?#{DOMAIN_VALID_CHARS}\.)/io
|
|
169
190
|
REGEXEN[:valid_domain_name] = /(?:(?:#{DOMAIN_VALID_CHARS}(?:[-]|#{DOMAIN_VALID_CHARS})*)?#{DOMAIN_VALID_CHARS}\.)/io
|
|
170
191
|
|
|
@@ -183,13 +204,6 @@ module Twitter
|
|
|
183
204
|
}ix
|
|
184
205
|
REGEXEN[:valid_punycode] = /(?:xn--[0-9a-z]+)/i
|
|
185
206
|
|
|
186
|
-
REGEXEN[:valid_special_cctld] = %r{
|
|
187
|
-
(?:
|
|
188
|
-
(?:co|tv)
|
|
189
|
-
(?=[^0-9a-z@]|$)
|
|
190
|
-
)
|
|
191
|
-
}ix
|
|
192
|
-
|
|
193
207
|
REGEXEN[:valid_domain] = /(?:
|
|
194
208
|
#{REGEXEN[:valid_subdomain]}*#{REGEXEN[:valid_domain_name]}
|
|
195
209
|
(?:#{REGEXEN[:valid_gTLD]}|#{REGEXEN[:valid_ccTLD]}|#{REGEXEN[:valid_punycode]})
|
|
@@ -204,10 +218,6 @@ module Twitter
|
|
|
204
218
|
# This is used in Extractor for stricter t.co URL extraction
|
|
205
219
|
REGEXEN[:valid_tco_url] = /^https?:\/\/t\.co\/([a-z0-9]+)/i
|
|
206
220
|
|
|
207
|
-
# This is used in Extractor to filter out unwanted URLs.
|
|
208
|
-
REGEXEN[:invalid_short_domain] = /\A#{REGEXEN[:valid_domain_name]}#{REGEXEN[:valid_ccTLD]}\Z/io
|
|
209
|
-
REGEXEN[:valid_special_short_domain] = /\A#{REGEXEN[:valid_domain_name]}#{REGEXEN[:valid_special_cctld]}\Z/io
|
|
210
|
-
|
|
211
221
|
REGEXEN[:valid_port_number] = /[0-9]+/
|
|
212
222
|
|
|
213
223
|
REGEXEN[:valid_general_url_path_chars] = /[a-z\p{Cyrillic}0-9!\*';:=\+\,\.\$\/%#\[\]\p{Pd}_~&\|@#{LATIN_ACCENTS}]/io
|
|
@@ -258,7 +268,7 @@ module Twitter
|
|
|
258
268
|
}iox
|
|
259
269
|
|
|
260
270
|
REGEXEN[:cashtag] = /[a-z]{1,6}(?:[._][a-z]{1,2})?/i
|
|
261
|
-
REGEXEN[:valid_cashtag] = /(
|
|
271
|
+
REGEXEN[:valid_cashtag] = /(^|[#{UNICODE_SPACES}#{DIRECTIONAL_CHARACTERS}])(\$)(#{REGEXEN[:cashtag]})(?=$|\s|[#{PUNCTUATION_CHARS}])/i
|
|
262
272
|
|
|
263
273
|
# These URL validation pattern strings are based on the ABNF from RFC 3986
|
|
264
274
|
REGEXEN[:validate_url_unreserved] = /[a-z\p{Cyrillic}0-9\p{Pd}._~]/i
|
|
@@ -339,6 +349,8 @@ module Twitter
|
|
|
339
349
|
REGEXEN[:validate_url_query] = %r{(#{REGEXEN[:validate_url_pchar]}|/|\?)*}i
|
|
340
350
|
REGEXEN[:validate_url_fragment] = %r{(#{REGEXEN[:validate_url_pchar]}|/|\?)*}i
|
|
341
351
|
|
|
352
|
+
REGEXEN[:valid_emoji] = Twitter::TwitterText::Regex::Emoji[:valid_emoji]
|
|
353
|
+
|
|
342
354
|
# Modified version of RFC 3986 Appendix B
|
|
343
355
|
REGEXEN[:validate_url_unencoded] = %r{
|
|
344
356
|
\A # Full URL
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
module Twitter
|
|
2
6
|
module TwitterText
|
|
3
7
|
# A module provides base methods to rewrite usernames, lists, hashtags and URLs.
|
|
4
8
|
module Rewriter extend self
|
|
5
9
|
def rewrite_entities(text, entities)
|
|
6
|
-
|
|
10
|
+
codepoints = text.to_s.to_codepoint_a
|
|
7
11
|
|
|
8
12
|
# sort by start index
|
|
9
13
|
entities = entities.sort_by do |entity|
|
|
@@ -14,11 +18,11 @@ module Twitter
|
|
|
14
18
|
result = []
|
|
15
19
|
last_index = entities.inject(0) do |index, entity|
|
|
16
20
|
indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
|
|
17
|
-
result <<
|
|
18
|
-
result << yield(entity,
|
|
21
|
+
result << codepoints[index...indices.first]
|
|
22
|
+
result << yield(entity, codepoints)
|
|
19
23
|
indices.last
|
|
20
24
|
end
|
|
21
|
-
result <<
|
|
25
|
+
result << codepoints[last_index..-1]
|
|
22
26
|
|
|
23
27
|
result.flatten.join
|
|
24
28
|
end
|
|
@@ -35,8 +39,8 @@ module Twitter
|
|
|
35
39
|
|
|
36
40
|
def rewrite_usernames_or_lists(text)
|
|
37
41
|
entities = Extractor.extract_mentions_or_lists_with_indices(text)
|
|
38
|
-
rewrite_entities(text, entities) do |entity,
|
|
39
|
-
at =
|
|
42
|
+
rewrite_entities(text, entities) do |entity, codepoints|
|
|
43
|
+
at = codepoints[entity[:indices].first]
|
|
40
44
|
list_slug = entity[:list_slug]
|
|
41
45
|
list_slug = nil if list_slug.empty?
|
|
42
46
|
yield(at, entity[:screen_name], list_slug)
|
|
@@ -46,8 +50,8 @@ module Twitter
|
|
|
46
50
|
|
|
47
51
|
def rewrite_hashtags(text)
|
|
48
52
|
entities = Extractor.extract_hashtags_with_indices(text)
|
|
49
|
-
rewrite_entities(text, entities) do |entity,
|
|
50
|
-
hash =
|
|
53
|
+
rewrite_entities(text, entities) do |entity, codepoints|
|
|
54
|
+
hash = codepoints[entity[:indices].first]
|
|
51
55
|
yield(hash, entity[:hashtag])
|
|
52
56
|
end
|
|
53
57
|
end
|
|
@@ -55,7 +59,7 @@ module Twitter
|
|
|
55
59
|
|
|
56
60
|
def rewrite_urls(text)
|
|
57
61
|
entities = Extractor.extract_urls_with_indices(text, :extract_url_without_protocol => false)
|
|
58
|
-
rewrite_entities(text, entities) do |entity,
|
|
62
|
+
rewrite_entities(text, entities) do |entity, codepoints|
|
|
59
63
|
yield(entity[:url])
|
|
60
64
|
end
|
|
61
65
|
end
|
data/lib/twitter-text/unicode.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
require 'unf'
|
|
2
6
|
|
|
3
7
|
module Twitter
|
|
@@ -34,8 +38,7 @@ module Twitter
|
|
|
34
38
|
options = DEFAULT_TCO_URL_LENGTHS.merge(options)
|
|
35
39
|
config = options[:config] || Twitter::TwitterText::Configuration.default_configuration
|
|
36
40
|
normalized_text = text.to_nfc
|
|
37
|
-
|
|
38
|
-
unless (normalized_text_length > 0)
|
|
41
|
+
unless (normalized_text.length > 0)
|
|
39
42
|
ParseResults.empty()
|
|
40
43
|
end
|
|
41
44
|
|
|
@@ -46,6 +49,7 @@ module Twitter
|
|
|
46
49
|
ranges = config.ranges
|
|
47
50
|
|
|
48
51
|
url_entities = Twitter::TwitterText::Extractor.extract_urls_with_indices(normalized_text)
|
|
52
|
+
emoji_entities = config.emoji_parsing_enabled ? Twitter::TwitterText::Extractor.extract_emoji_with_indices(normalized_text) : []
|
|
49
53
|
|
|
50
54
|
has_invalid_chars = false
|
|
51
55
|
weighted_count = 0
|
|
@@ -53,24 +57,42 @@ module Twitter
|
|
|
53
57
|
display_offset = 0
|
|
54
58
|
valid_offset = 0
|
|
55
59
|
|
|
56
|
-
while offset <
|
|
60
|
+
while offset < normalized_text.codepoint_length
|
|
57
61
|
# Reset the default char weight each pass through the loop
|
|
58
62
|
char_weight = config.default_weight
|
|
63
|
+
entity_length = 0
|
|
64
|
+
|
|
59
65
|
url_entities.each do |url_entity|
|
|
60
66
|
if url_entity[:indices].first == offset
|
|
61
|
-
|
|
67
|
+
entity_length = url_entity[:indices].last - url_entity[:indices].first
|
|
62
68
|
weighted_count += transformed_url_length
|
|
63
|
-
offset +=
|
|
64
|
-
display_offset +=
|
|
69
|
+
offset += entity_length
|
|
70
|
+
display_offset += entity_length
|
|
71
|
+
if weighted_count <= scaled_max_weighted_tweet_length
|
|
72
|
+
valid_offset += entity_length
|
|
73
|
+
end
|
|
74
|
+
# Finding a match breaks the loop
|
|
75
|
+
break
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
emoji_entities.each do |emoji_entity|
|
|
80
|
+
if emoji_entity[:indices].first == offset
|
|
81
|
+
entity_length = emoji_entity[:indices].last - emoji_entity[:indices].first
|
|
82
|
+
weighted_count += char_weight # the default weight
|
|
83
|
+
offset += entity_length
|
|
84
|
+
display_offset += entity_length
|
|
65
85
|
if weighted_count <= scaled_max_weighted_tweet_length
|
|
66
|
-
valid_offset +=
|
|
86
|
+
valid_offset += entity_length
|
|
67
87
|
end
|
|
68
|
-
# Finding a match breaks the loop
|
|
88
|
+
# Finding a match breaks the loop
|
|
69
89
|
break
|
|
70
90
|
end
|
|
71
91
|
end
|
|
72
92
|
|
|
73
|
-
if
|
|
93
|
+
next if entity_length > 0
|
|
94
|
+
|
|
95
|
+
if offset < normalized_text.codepoint_length
|
|
74
96
|
code_point = normalized_text[offset]
|
|
75
97
|
|
|
76
98
|
ranges.each do |range|
|
|
@@ -82,17 +104,19 @@ module Twitter
|
|
|
82
104
|
|
|
83
105
|
weighted_count += char_weight
|
|
84
106
|
|
|
85
|
-
has_invalid_chars = contains_invalid?(
|
|
86
|
-
|
|
87
|
-
offset +=
|
|
88
|
-
display_offset +=
|
|
107
|
+
has_invalid_chars = contains_invalid?(code_point) unless has_invalid_chars
|
|
108
|
+
codepoint_length = code_point.codepoint_length
|
|
109
|
+
offset += codepoint_length
|
|
110
|
+
display_offset += codepoint_length
|
|
111
|
+
# index += codepoint_length
|
|
89
112
|
|
|
90
113
|
if !has_invalid_chars && (weighted_count <= scaled_max_weighted_tweet_length)
|
|
91
|
-
valid_offset +=
|
|
114
|
+
valid_offset += codepoint_length
|
|
92
115
|
end
|
|
93
116
|
end
|
|
94
117
|
end
|
|
95
|
-
|
|
118
|
+
|
|
119
|
+
normalized_text_offset = text.codepoint_length - normalized_text.codepoint_length
|
|
96
120
|
scaled_weighted_length = weighted_count / scale
|
|
97
121
|
is_valid = !has_invalid_chars && (scaled_weighted_length <= max_weighted_tweet_length)
|
|
98
122
|
permillage = scaled_weighted_length * 1000 / max_weighted_tweet_length
|
data/spec/autolinking_spec.rb
CHANGED
data/spec/configuration_spec.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
require File.dirname(__FILE__) + '/spec_helper'
|
|
3
7
|
|
|
@@ -18,6 +22,7 @@ describe Twitter::TwitterText::Configuration do
|
|
|
18
22
|
it "should define version constants" do
|
|
19
23
|
expect(Twitter::TwitterText::Configuration.const_defined?(:CONFIG_V1)).to be true
|
|
20
24
|
expect(Twitter::TwitterText::Configuration.const_defined?(:CONFIG_V2)).to be true
|
|
25
|
+
expect(Twitter::TwitterText::Configuration.const_defined?(:CONFIG_V3)).to be true
|
|
21
26
|
end
|
|
22
27
|
|
|
23
28
|
it "should define a default configuration" do
|
|
@@ -87,5 +92,45 @@ describe Twitter::TwitterText::Configuration do
|
|
|
87
92
|
expect(weighted_range.weight).to be_kind_of(Integer)
|
|
88
93
|
end
|
|
89
94
|
end
|
|
95
|
+
|
|
96
|
+
context "with v3 configuration" do
|
|
97
|
+
before do
|
|
98
|
+
@config = Twitter::TwitterText::Configuration.configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V3)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
it "should have a version" do
|
|
102
|
+
expect(@config.version).to eq(3)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it "should have a max_weighted_tweet_length" do
|
|
106
|
+
expect(@config.max_weighted_tweet_length).to eq(280)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
it "should have a scale" do
|
|
110
|
+
expect(@config.scale).to eq(100)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
it "should have a default_weight" do
|
|
114
|
+
expect(@config.default_weight).to eq(200)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
it "should have a transformed_url_length" do
|
|
118
|
+
expect(@config.transformed_url_length).to eq(23)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it "should have a configured range" do
|
|
122
|
+
expect(@config.ranges).to be_kind_of(Array)
|
|
123
|
+
expect(@config.ranges.count).to be > 0
|
|
124
|
+
expect(@config.ranges[0]).to be_kind_of(Twitter::TwitterText::WeightedRange)
|
|
125
|
+
weighted_range = @config.ranges[0]
|
|
126
|
+
expect(weighted_range.start).to be_kind_of(Integer)
|
|
127
|
+
expect(weighted_range.end).to be_kind_of(Integer)
|
|
128
|
+
expect(weighted_range.weight).to be_kind_of(Integer)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
it "should support discounting emoji" do
|
|
132
|
+
expect(@config.emoji_parsing_enabled).to be true
|
|
133
|
+
end
|
|
134
|
+
end
|
|
90
135
|
end
|
|
91
136
|
end
|
data/spec/extractor_spec.rb
CHANGED
data/spec/hithighlighter_spec.rb
CHANGED
data/spec/regex_spec.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
require File.dirname(__FILE__) + '/spec_helper'
|
|
3
7
|
|
|
@@ -35,4 +39,38 @@ describe "Twitter::TwitterText::Regex regular expressions" do
|
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
end
|
|
42
|
+
|
|
43
|
+
describe "matching Unicode 10.0 emoji" do
|
|
44
|
+
it "should match new emoji" do
|
|
45
|
+
input = "Unicode 10.0; grinning face with one large and one small eye: 🤪; woman with headscarf: 🧕; (fitzpatrick) woman with headscarf + medium-dark skin tone: 🧕🏾; flag (England): 🏴"
|
|
46
|
+
expected = ["🤪", "🧕", "🧕🏾", "🏴"]
|
|
47
|
+
entities = Twitter::TwitterText::Extractor.extract_emoji_with_indices(input)
|
|
48
|
+
entities.each_with_index do |entity, i|
|
|
49
|
+
expect(entity[:emoji]).to be_kind_of(String)
|
|
50
|
+
expect(entity[:indices]).to be_kind_of(Array)
|
|
51
|
+
entity[:indices].each do |position|
|
|
52
|
+
expect(position).to be_kind_of(Integer)
|
|
53
|
+
end
|
|
54
|
+
expect(entity[:emoji]).to be == expected[i]
|
|
55
|
+
expect(Twitter::TwitterText::Extractor.is_valid_emoji(entity[:emoji])).to be true
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
describe "matching Unicode 9.0 emoji" do
|
|
61
|
+
it "should match new emoji" do
|
|
62
|
+
input = "Unicode 9.0; face with cowboy hat: 🤠; woman dancing: 💃, woman dancing + medium-dark skin tone: 💃🏾"
|
|
63
|
+
expected = ["🤠", "💃", "💃🏾"]
|
|
64
|
+
entities = Twitter::TwitterText::Extractor.extract_emoji_with_indices(input)
|
|
65
|
+
entities.each_with_index do |entity, i|
|
|
66
|
+
expect(entity[:emoji]).to be_kind_of(String)
|
|
67
|
+
expect(entity[:indices]).to be_kind_of(Array)
|
|
68
|
+
entity[:indices].each do |position|
|
|
69
|
+
expect(position).to be_kind_of(Integer)
|
|
70
|
+
end
|
|
71
|
+
expect(entity[:emoji]).to be == expected[i]
|
|
72
|
+
expect(Twitter::TwitterText::Extractor.is_valid_emoji(entity[:emoji])).to be true
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
38
76
|
end
|
data/spec/rewriter_spec.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
$TESTING=true
|
|
2
6
|
|
|
3
7
|
# Ruby 1.8 encoding check
|
|
@@ -20,6 +24,15 @@ require File.expand_path('../test_urls', __FILE__)
|
|
|
20
24
|
|
|
21
25
|
RSpec.configure do |config|
|
|
22
26
|
config.include TestUrls
|
|
27
|
+
|
|
28
|
+
config.filter_run_excluding :ruby => lambda { |version|
|
|
29
|
+
case version.to_s
|
|
30
|
+
when /^> (.*)/
|
|
31
|
+
!(RUBY_VERSION.to_s > $1)
|
|
32
|
+
else
|
|
33
|
+
!(RUBY_VERSION.to_s =~ /^#{version.to_s}/)
|
|
34
|
+
end
|
|
35
|
+
}
|
|
23
36
|
end
|
|
24
37
|
|
|
25
38
|
RSpec::Matchers.define :match_autolink_expression do
|
data/spec/test_urls.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
|
|
3
7
|
module TestUrls
|
|
@@ -53,11 +57,11 @@ module TestUrls
|
|
|
53
57
|
"http://no_underscores.com",
|
|
54
58
|
"http://test.c_o_m",
|
|
55
59
|
"http://test.c-o-m",
|
|
56
|
-
"http://twitt#{[0x202A].pack('U')}
|
|
57
|
-
"http://twitt#{[0x202B].pack('U')}
|
|
58
|
-
"http://twitt#{[0x202C].pack('U')}
|
|
59
|
-
"http://twitt#{[0x202D].pack('U')}
|
|
60
|
-
"http://twitt#{[0x202E].pack('U')}
|
|
60
|
+
"http://twitt#{[0x202A].pack('U')}.com",
|
|
61
|
+
"http://twitt#{[0x202B].pack('U')}.com",
|
|
62
|
+
"http://twitt#{[0x202C].pack('U')}.com",
|
|
63
|
+
"http://twitt#{[0x202D].pack('U')}.com",
|
|
64
|
+
"http://twitt#{[0x202E].pack('U')}.com",
|
|
61
65
|
"https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo https://somesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurlsomesuperlongurl.com/foo"
|
|
62
66
|
] unless defined?(TestUrls::INVALID)
|
|
63
67
|
|
data/spec/twitter_text_spec.rb
CHANGED
data/spec/unicode_spec.rb
CHANGED
data/spec/validation_spec.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
require File.dirname(__FILE__) + '/spec_helper'
|
|
3
7
|
|
|
@@ -16,9 +20,9 @@ describe Twitter::TwitterText::Validation do
|
|
|
16
20
|
expect(TestValidation.new.tweet_invalid?("Bom:#{Twitter::TwitterText::Unicode::UFFFF}")).to be == :invalid_characters
|
|
17
21
|
end
|
|
18
22
|
|
|
19
|
-
it "should
|
|
23
|
+
it "should allow direction change characters" do
|
|
20
24
|
[0x202A, 0x202B, 0x202C, 0x202D, 0x202E].map{|cp| [cp].pack('U') }.each do |char|
|
|
21
|
-
expect(TestValidation.new.tweet_invalid?("Invalid:#{char}")).to
|
|
25
|
+
expect(TestValidation.new.tweet_invalid?("Invalid:#{char}")).to be false
|
|
22
26
|
end
|
|
23
27
|
end
|
|
24
28
|
|
|
@@ -64,4 +68,20 @@ describe Twitter::TwitterText::Validation do
|
|
|
64
68
|
expect(results[:valid_range_end]).to eq(0)
|
|
65
69
|
end
|
|
66
70
|
end
|
|
71
|
+
|
|
72
|
+
context "when parsing tweet text" do
|
|
73
|
+
it "should properly parse ZWJ and ZWNJ when grapheme clusters are enabled", ruby: ">= 2.5.0" do
|
|
74
|
+
# Grapheme clustering of devenghali script differs based on platform implementation
|
|
75
|
+
text = "ZWJ: क्ष -> क्\u200Dष; ZWNJ: क्ष -> क्\u200Cष"
|
|
76
|
+
config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V3)
|
|
77
|
+
results = Twitter::TwitterText::Validation::parse_tweet(text, config: config)
|
|
78
|
+
expect(results[:weighted_length]).to eq(29)
|
|
79
|
+
expect(results[:permillage]).to eq(103)
|
|
80
|
+
expect(results[:valid]).to be true
|
|
81
|
+
expect(results[:display_range_start]).to eq(0)
|
|
82
|
+
expect(results[:display_range_end]).to eq(34)
|
|
83
|
+
expect(results[:valid_range_start]).to eq(0)
|
|
84
|
+
expect(results[:valid_range_end]).to eq(34)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
67
87
|
end
|
data/test/conformance_test.rb
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
require 'multi_json'
|
|
2
6
|
require 'nokogiri'
|
|
3
7
|
require 'test/unit'
|
|
@@ -130,6 +134,11 @@ class ConformanceTest < Test::Unit::TestCase
|
|
|
130
134
|
assert_equal e, extract_urls_with_indices(text), description
|
|
131
135
|
end
|
|
132
136
|
|
|
137
|
+
def_conformance_test("extract.yml", :urls_with_directional_markers) do
|
|
138
|
+
e = expected.map{|elem| elem.inject({}){|h, (k,v)| h[k.to_sym] = v; h} }
|
|
139
|
+
assert_equal e, extract_urls_with_indices(text), description
|
|
140
|
+
end
|
|
141
|
+
|
|
133
142
|
def_conformance_test("extract.yml", :hashtags) do
|
|
134
143
|
assert_equal expected, extract_hashtags(text), description
|
|
135
144
|
end
|
|
@@ -215,11 +224,19 @@ class ConformanceTest < Test::Unit::TestCase
|
|
|
215
224
|
assert_equal expected, valid_hashtag?(text), description
|
|
216
225
|
end
|
|
217
226
|
|
|
218
|
-
def_conformance_test("validate.yml", :
|
|
219
|
-
|
|
227
|
+
def_conformance_test("validate.yml", :WeightedTweetsCounterTest) do
|
|
228
|
+
# Force v2 configuration, basic weighted code point support
|
|
229
|
+
config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V2)
|
|
230
|
+
assert_equal_parse_results expected, parse_tweet(text, config: config), description
|
|
220
231
|
end
|
|
221
232
|
|
|
222
|
-
def_conformance_test("validate.yml", :
|
|
233
|
+
def_conformance_test("validate.yml", :WeightedTweetsWithDiscountedEmojiCounterTest) do
|
|
234
|
+
# Force v3 configuration, which supports discounting grapheme clusters that are emoji
|
|
235
|
+
config = Twitter::TwitterText::Configuration::configuration_from_file(Twitter::TwitterText::Configuration::CONFIG_V3)
|
|
236
|
+
assert_equal_parse_results expected, parse_tweet(text, config: config), description
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def_conformance_test("validate.yml", :UnicodeDirectionalMarkerCounterTest) do
|
|
223
240
|
assert_equal_parse_results expected, parse_tweet(text), description
|
|
224
241
|
end
|
|
225
242
|
end
|
data/twitter-text.gemspec
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
+
# Copyright 2018 Twitter, Inc.
|
|
2
|
+
# Licensed under the Apache License, Version 2.0
|
|
3
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
4
|
+
|
|
1
5
|
# encoding: utf-8
|
|
2
6
|
|
|
3
7
|
Gem::Specification.new do |s|
|
|
4
8
|
s.name = "twitter-text"
|
|
5
|
-
s.version = "
|
|
9
|
+
s.version = "3.0.0"
|
|
6
10
|
s.authors = ["David LaMacchia", "Sudheer Guntupalli", "Kaushik Lakshmikanth", "Jose Antonio Marquez Russo", "Lee Adams",
|
|
7
11
|
"Yoshimasa Niwa"]
|
|
8
12
|
s.email = ["opensource@twitter.com"]
|
|
@@ -14,7 +18,6 @@ Gem::Specification.new do |s|
|
|
|
14
18
|
s.has_rdoc = true
|
|
15
19
|
s.summary = "Twitter text handling library"
|
|
16
20
|
|
|
17
|
-
s.add_development_dependency "pry"
|
|
18
21
|
s.add_development_dependency "test-unit"
|
|
19
22
|
s.add_development_dependency "multi_json", "~> 1.3"
|
|
20
23
|
s.add_development_dependency "nokogiri", "~> 1.8.0"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: twitter-text
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- David LaMacchia
|
|
@@ -13,22 +13,8 @@ authors:
|
|
|
13
13
|
autorequire:
|
|
14
14
|
bindir: bin
|
|
15
15
|
cert_chain: []
|
|
16
|
-
date:
|
|
16
|
+
date: 2018-10-14 00:00:00.000000000 Z
|
|
17
17
|
dependencies:
|
|
18
|
-
- !ruby/object:Gem::Dependency
|
|
19
|
-
name: pry
|
|
20
|
-
requirement: !ruby/object:Gem::Requirement
|
|
21
|
-
requirements:
|
|
22
|
-
- - ">="
|
|
23
|
-
- !ruby/object:Gem::Version
|
|
24
|
-
version: '0'
|
|
25
|
-
type: :development
|
|
26
|
-
prerelease: false
|
|
27
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
28
|
-
requirements:
|
|
29
|
-
- - ">="
|
|
30
|
-
- !ruby/object:Gem::Version
|
|
31
|
-
version: '0'
|
|
32
18
|
- !ruby/object:Gem::Dependency
|
|
33
19
|
name: test-unit
|
|
34
20
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -174,11 +160,13 @@ files:
|
|
|
174
160
|
- config/README.md
|
|
175
161
|
- config/v1.json
|
|
176
162
|
- config/v2.json
|
|
163
|
+
- config/v3.json
|
|
177
164
|
- lib/assets/tld_lib.yml
|
|
178
165
|
- lib/twitter-text.rb
|
|
179
166
|
- lib/twitter-text/autolink.rb
|
|
180
167
|
- lib/twitter-text/configuration.rb
|
|
181
168
|
- lib/twitter-text/deprecation.rb
|
|
169
|
+
- lib/twitter-text/emoji_regex.rb
|
|
182
170
|
- lib/twitter-text/extractor.rb
|
|
183
171
|
- lib/twitter-text/hash_helper.rb
|
|
184
172
|
- lib/twitter-text/hit_highlighter.rb
|
|
@@ -222,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
222
210
|
version: '0'
|
|
223
211
|
requirements: []
|
|
224
212
|
rubyforge_project:
|
|
225
|
-
rubygems_version: 2.
|
|
213
|
+
rubygems_version: 2.5.1
|
|
226
214
|
signing_key:
|
|
227
215
|
specification_version: 4
|
|
228
216
|
summary: Twitter text handling library
|