uts58 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -7
- data/lib/uts58/extractor.rb +4 -4
- data/lib/uts58/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 29a080a46f1b16f0a6915c30697abb9e91822afa8331572a2652ab5c6f0f5996
|
|
4
|
+
data.tar.gz: 82f302762d3aa0d15198078370145081bb027d646f3638b70a0703b1bdfbaa29
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 579b151954c46e0bbb425a0939c6a9210e2fa1186f8748e38bae94d8a98a60a67c897fa4a50b0348d2ac6b1fb9a5213ad3babf7fbe818b12d22519a6a6b6606b
|
|
7
|
+
data.tar.gz: bf69e940d10a4d2bf93b76d763342022a92f543622f4c5bbdf48748e3c6a38ecdb56f473e036d979cc974af15f7fb70a452aac5a607a4d004fbe584d8d717faf
|
data/README.md
CHANGED
|
@@ -5,8 +5,9 @@ the Unicode spec for finding links in running text. Given a chunk of text,
|
|
|
5
5
|
it returns the URLs and email addresses in it along with their character
|
|
6
6
|
offsets.
|
|
7
7
|
|
|
8
|
-
Both halves of UTS58 are covered:
|
|
9
|
-
The two are detected independently and can be combined.
|
|
8
|
+
Both halves of UTS58 are covered: web links and email addresses.
|
|
9
|
+
The two are detected independently and can be combined. You can also
|
|
10
|
+
combine them with detection of your own, e.g. usernames on your service.
|
|
10
11
|
|
|
11
12
|
Tested extensively on relevant OSes: [](https://github.com/arnt/uts58/actions/workflows/ruby.yml)
|
|
12
13
|
|
|
@@ -59,7 +60,7 @@ are handled per the spec.
|
|
|
59
60
|
## Email addresses
|
|
60
61
|
|
|
61
62
|
Email detection mirrors the URL methods. Each result carries the address
|
|
62
|
-
twice
|
|
63
|
+
twice (as a bare `:email` and as a `mailto:` `:url`) so it drops straight
|
|
63
64
|
into anything that already renders a `:url` entity:
|
|
64
65
|
|
|
65
66
|
```ruby
|
|
@@ -79,7 +80,7 @@ are recognised; the domain is IDN-decoded just like a URL host. A leading
|
|
|
79
80
|
## Combined extraction
|
|
80
81
|
|
|
81
82
|
`extract_entities_with_indices` runs both detectors, sorts by offset, and
|
|
82
|
-
strips overlaps
|
|
83
|
+
strips overlaps, very similar to `Twitter::TwitterText::Extractor#extract_entities_with_indices`.
|
|
83
84
|
The result is a mixed list of `:url` and email (`:email` + `:url`) hashes:
|
|
84
85
|
|
|
85
86
|
```ruby
|
|
@@ -100,7 +101,7 @@ you have two options, with different results for `contact info@example.com for p
|
|
|
100
101
|
1. **Extract both, then drop emails.** Take `extract_entities_with_indices`
|
|
101
102
|
(already overlap-stripped) and reject the hashes that have an `:email`
|
|
102
103
|
key. The address wins the overlap, so dropping it leaves that span
|
|
103
|
-
*unlinked* — `info@example.com` stays plain text. Choose this if an
|
|
104
|
+
*unlinked* — `contact info@example.com` stays plain text. Choose this if an
|
|
104
105
|
address shouldn't silently become a website link.
|
|
105
106
|
2. **Extract only URLs.** Call `extract_urls_with_indices` and skip email
|
|
106
107
|
detection entirely. The URL scan finds domain after the `@`, so the
|
|
@@ -110,7 +111,10 @@ you have two options, with different results for `contact info@example.com for p
|
|
|
110
111
|
## What's not here
|
|
111
112
|
|
|
112
113
|
- **Link validation.** Recognised URLs are not fetched, normalised beyond
|
|
113
|
-
IDN decoding, or their hostnames checked in the DNS.
|
|
114
|
+
IDN decoding, or their hostnames checked in the DNS. There is
|
|
115
|
+
no attempt at checking for possible attacks (`Мышκин.рф` and
|
|
116
|
+
`Мышкин.рф` are both detected, note the Greek kappa in the middle of
|
|
117
|
+
the prince's name). If you need this,
|
|
114
118
|
send me mail.
|
|
115
119
|
|
|
116
120
|
## Roadmap
|
|
@@ -118,7 +122,7 @@ you have two options, with different results for `contact info@example.com for p
|
|
|
118
122
|
My immediate need is UTS58 conformant link detection suitable for
|
|
119
123
|
public web pages. If you need something more, I rather think that an
|
|
120
124
|
item can be added to this roadmap, so long as the description in the
|
|
121
|
-
rdoc remains short and simple. Send mail
|
|
125
|
+
rdoc remains short and simple. Send me mail.
|
|
122
126
|
|
|
123
127
|
## License
|
|
124
128
|
|
data/lib/uts58/extractor.rb
CHANGED
|
@@ -77,7 +77,7 @@ module Uts58
|
|
|
77
77
|
# nothing precedes it, and the actual link begins three
|
|
78
78
|
# codepoints later.
|
|
79
79
|
s = match.post_match
|
|
80
|
-
scheme_match =
|
|
80
|
+
scheme_match = /\A([\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}\p{Thai}\p{Lao}\p{Khmer}\p{Myanmar}]*?)(https?:\/\/)/i.match(s)
|
|
81
81
|
if scheme_match
|
|
82
82
|
scheme_offset = scheme_match[1].length
|
|
83
83
|
proto = scheme_match[2]
|
|
@@ -88,7 +88,7 @@ module Uts58
|
|
|
88
88
|
end
|
|
89
89
|
# look for the prefix that might be a hostname or an IDN.
|
|
90
90
|
# this is a somewhat sloppy match, with a few false positives.
|
|
91
|
-
prefix =
|
|
91
|
+
prefix = /\A([-\p{L}\p{N}\p{M}\u00DF\u03C2\u06FD\u06FE\u0F0B\u3007]+[\.。]){1,4}[-\p{L}\p{N}\p{M}]+(?![-\p{L}\p{N}\p{M}])/.match(s)
|
|
92
92
|
if prefix && prefix[0].length < 254
|
|
93
93
|
host_raw = prefix.match(0).gsub(/。/, ".")
|
|
94
94
|
next unless valid_labels?(host_raw)
|
|
@@ -108,7 +108,7 @@ module Uts58
|
|
|
108
108
|
rest = rest[1..]
|
|
109
109
|
end
|
|
110
110
|
# a port number must be 1..65535
|
|
111
|
-
port =
|
|
111
|
+
port = /\A:(\d+)/.match(rest)
|
|
112
112
|
if port
|
|
113
113
|
n = port[1].to_i
|
|
114
114
|
next if n < 1 || n > 65535
|
|
@@ -180,7 +180,7 @@ module Uts58
|
|
|
180
180
|
local = lp_match[0]
|
|
181
181
|
next if local.start_with?('.') || local.end_with?('.') || local.include?('..')
|
|
182
182
|
s = match.post_match
|
|
183
|
-
prefix =
|
|
183
|
+
prefix = /\A([-\p{L}\p{N}\p{M}ßς۽۾་〇]+[\.。]){1,4}[-\p{L}\p{N}\p{M}]+(?![-\p{L}\p{N}\p{M}])/.match(s)
|
|
184
184
|
next unless prefix && prefix[0].length < 254
|
|
185
185
|
host_raw = prefix.match(0).gsub(/。/, ".")
|
|
186
186
|
next unless valid_labels?(host_raw)
|
data/lib/uts58/version.rb
CHANGED