textprov 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +202 -19
- data/README.md +83 -47
- data/lib/textprov/cli.rb +14 -7
- data/lib/textprov/core.rb +53 -74
- data/lib/textprov/display.rb +10 -0
- data/lib/textprov/encode.rb +46 -64
- data/lib/textprov/grapheme.rb +129 -0
- data/lib/textprov/mapping.json +2 -5
- data/lib/textprov/textprov.css +151 -0
- data/lib/textprov/ucd.rb +389 -0
- data/lib/textprov/version.rb +1 -1
- data/lib/textprov.rb +1 -0
- metadata +8 -5
data/lib/textprov/core.rb
CHANGED
|
@@ -4,56 +4,22 @@
|
|
|
4
4
|
|
|
5
5
|
require "cgi"
|
|
6
6
|
require "json"
|
|
7
|
+
require "set"
|
|
8
|
+
|
|
9
|
+
require_relative "grapheme"
|
|
7
10
|
|
|
8
11
|
module Textprov
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
SPEC_VERSION = "0.2" # Legacy API name; experimental baseline identifier.
|
|
13
|
+
UNICODE_VERSION = UCD::UNICODE_VERSION
|
|
14
|
+
GENERATED_STATES = %w[human ai].freeze
|
|
12
15
|
|
|
13
16
|
MAPPING_PATH = File.expand_path("mapping.json", __dir__)
|
|
14
17
|
|
|
15
|
-
ZWJ = 0x200D
|
|
16
|
-
VS15 = 0xFE0E
|
|
17
|
-
VS16 = 0xFE0F
|
|
18
|
-
SKIN_TONES = (0x1F3FB..0x1F3FF)
|
|
19
|
-
REGIONAL = (0x1F1E6..0x1F1FF)
|
|
20
|
-
|
|
21
|
-
# Categories Mn/Mc/Me: extracted from Unicode data. Ruby's regexp \p{Mn} etc.
|
|
22
|
-
# covers this; using a regex avoids shipping our own category table.
|
|
23
|
-
COMBINING_RE = /[\p{Mn}\p{Mc}\p{Me}]/
|
|
24
|
-
|
|
25
18
|
module_function
|
|
26
19
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
# A cluster is a base character plus everything that visually belongs to it:
|
|
32
|
-
# combining marks, VS15/VS16, skin-tone modifiers, ZWJ joins, and the second
|
|
33
|
-
# half of a regional-indicator pair. A provenance selector always ends the
|
|
34
|
-
# cluster.
|
|
35
|
-
def cluster_end(chars, start, sel_cps)
|
|
36
|
-
index = start + 1
|
|
37
|
-
if REGIONAL.include?(chars[start].ord) &&
|
|
38
|
-
index < chars.length &&
|
|
39
|
-
REGIONAL.include?(chars[index].ord)
|
|
40
|
-
index += 1
|
|
41
|
-
end
|
|
42
|
-
while index < chars.length
|
|
43
|
-
cp = chars[index].ord
|
|
44
|
-
break if sel_cps.include?(cp)
|
|
45
|
-
|
|
46
|
-
if cp == ZWJ
|
|
47
|
-
index += (index + 1 < chars.length ? 2 : 1)
|
|
48
|
-
next
|
|
49
|
-
end
|
|
50
|
-
if cp == VS15 || cp == VS16 || SKIN_TONES.include?(cp) || is_combining(chars[index])
|
|
51
|
-
index += 1
|
|
52
|
-
next
|
|
53
|
-
end
|
|
54
|
-
break
|
|
55
|
-
end
|
|
56
|
-
index
|
|
20
|
+
# Split `text` into extended grapheme clusters, the units a selector marks.
|
|
21
|
+
def segments(text)
|
|
22
|
+
Grapheme.segments(text)
|
|
57
23
|
end
|
|
58
24
|
|
|
59
25
|
class Mapping
|
|
@@ -97,6 +63,11 @@ module Textprov
|
|
|
97
63
|
attr_writer :default_mapping
|
|
98
64
|
end
|
|
99
65
|
|
|
66
|
+
# Exact Unicode White_Space set used by the experiment, independent of runtime predicates.
|
|
67
|
+
def blank?(text)
|
|
68
|
+
text.match?(/\A[\u0009-\u000D\u0020\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]+\z/)
|
|
69
|
+
end
|
|
70
|
+
|
|
100
71
|
# `text` is a String; we work over its per-codepoint characters. Ruby's
|
|
101
72
|
# String#chars yields one Unicode scalar per element, which is what the
|
|
102
73
|
# Python port iterates.
|
|
@@ -113,34 +84,27 @@ module Textprov
|
|
|
113
84
|
out
|
|
114
85
|
end
|
|
115
86
|
|
|
87
|
+
# Implements the decoder algorithm in the baseline experiment: segment into
|
|
88
|
+
# extended grapheme clusters, classify each cluster, then merge.
|
|
116
89
|
def _runs(text, selectors, pua2base, strip: false, merge_whitespace: true)
|
|
117
|
-
sel_cps = selectors.values.to_set
|
|
118
90
|
sel2name = selectors.each_with_object({}) { |(name, cp), h| h[cp] = name }
|
|
119
|
-
chars = text.chars
|
|
120
91
|
items = []
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
state = "ws"
|
|
131
|
-
out = char
|
|
132
|
-
index += 1
|
|
133
|
-
else
|
|
134
|
-
endi = cluster_end(chars, index, sel_cps)
|
|
135
|
-
cluster = chars[index...endi].join
|
|
136
|
-
state = nil
|
|
137
|
-
out = cluster
|
|
138
|
-
index = endi
|
|
139
|
-
if index < chars.length && sel_cps.include?(chars[index].ord)
|
|
140
|
-
state = sel2name[chars[index].ord]
|
|
141
|
-
out += chars[index] unless strip
|
|
142
|
-
index += 1
|
|
92
|
+
segments(text).each do |cluster|
|
|
93
|
+
last = cluster[-1].ord
|
|
94
|
+
state = nil
|
|
95
|
+
out = cluster
|
|
96
|
+
if sel2name.key?(last) && cluster.length > 1
|
|
97
|
+
# Whitespace cannot carry a mark; a selector after it is inert.
|
|
98
|
+
unless blank?(cluster[0...-1])
|
|
99
|
+
state = sel2name[last]
|
|
100
|
+
out = cluster[0...-1] if strip
|
|
143
101
|
end
|
|
102
|
+
elsif cluster.length == 1 && pua2base.key?(cluster.ord)
|
|
103
|
+
base_cp, state = pua2base[cluster.ord]
|
|
104
|
+
out = base_cp.chr(Encoding::UTF_8)
|
|
105
|
+
out += selectors[state].chr(Encoding::UTF_8) unless strip
|
|
106
|
+
elsif blank?(cluster)
|
|
107
|
+
state = "ws"
|
|
144
108
|
end
|
|
145
109
|
if !items.empty? && items.last[0] == state
|
|
146
110
|
items.last[1] << out
|
|
@@ -166,8 +130,23 @@ module Textprov
|
|
|
166
130
|
merged.map { |state, out| [state, out] }
|
|
167
131
|
end
|
|
168
132
|
|
|
169
|
-
def _to_html(text, selectors, pua2base, strip: false, merge_whitespace:
|
|
170
|
-
class_prefix: "prov")
|
|
133
|
+
def _to_html(text, selectors, pua2base, strip: false, merge_whitespace: false,
|
|
134
|
+
class_prefix: "prov", display: "passage")
|
|
135
|
+
raise ArgumentError, "display must be 'passage' or 'exact'" unless ["passage", "exact"].include?(display)
|
|
136
|
+
if display == "passage"
|
|
137
|
+
return text.split(/((?:\r?\n[ \t]*){2,}|\u2029)/, -1).map do |paragraph|
|
|
138
|
+
exact = _runs(paragraph, selectors, pua2base, strip: strip, merge_whitespace: false)
|
|
139
|
+
states = exact.map(&:first).compact.uniq.sort
|
|
140
|
+
content = _to_html(paragraph, selectors, pua2base, strip: strip,
|
|
141
|
+
merge_whitespace: false, class_prefix: class_prefix, display: "exact")
|
|
142
|
+
if states.empty?
|
|
143
|
+
content
|
|
144
|
+
else
|
|
145
|
+
kind = states.length == 1 ? states.first : "mixed"
|
|
146
|
+
%(<span class="#{class_prefix}-passage" data-prov-passage="#{kind}">#{content}</span>)
|
|
147
|
+
end
|
|
148
|
+
end.join
|
|
149
|
+
end
|
|
171
150
|
parts = []
|
|
172
151
|
_runs(text, selectors, pua2base, strip: strip,
|
|
173
152
|
merge_whitespace: merge_whitespace).each do |state, out|
|
|
@@ -181,8 +160,8 @@ module Textprov
|
|
|
181
160
|
parts.join
|
|
182
161
|
end
|
|
183
162
|
|
|
184
|
-
# Public API accepts either merge_whitespace (
|
|
185
|
-
# mergeWhitespace (camelCase alias per
|
|
163
|
+
# Public API accepts either merge_whitespace (snake_case spelling) or
|
|
164
|
+
# mergeWhitespace (camelCase alias per the experimental cross-language API).
|
|
186
165
|
def runs(text, mapping: nil, strip: false, merge_whitespace: nil, mergeWhitespace: nil)
|
|
187
166
|
mw = if merge_whitespace.nil?
|
|
188
167
|
mergeWhitespace.nil? || mergeWhitespace
|
|
@@ -194,15 +173,15 @@ module Textprov
|
|
|
194
173
|
end
|
|
195
174
|
|
|
196
175
|
def to_html(text, mapping: nil, strip: false, merge_whitespace: nil, mergeWhitespace: nil,
|
|
197
|
-
class_prefix: "prov")
|
|
176
|
+
class_prefix: "prov", display: "passage")
|
|
198
177
|
mw = if merge_whitespace.nil?
|
|
199
|
-
mergeWhitespace.nil?
|
|
178
|
+
mergeWhitespace.nil? ? false : mergeWhitespace
|
|
200
179
|
else
|
|
201
180
|
merge_whitespace
|
|
202
181
|
end
|
|
203
182
|
m = mapping || default_mapping
|
|
204
183
|
_to_html(text, m.selectors, m.pua2base, strip: strip, merge_whitespace: mw,
|
|
205
|
-
class_prefix: class_prefix)
|
|
184
|
+
class_prefix: class_prefix, display: display)
|
|
206
185
|
end
|
|
207
186
|
|
|
208
187
|
def strip_marks(text, mapping: nil)
|
data/lib/textprov/encode.rb
CHANGED
|
@@ -7,32 +7,26 @@ require_relative "core"
|
|
|
7
7
|
module Textprov
|
|
8
8
|
module_function
|
|
9
9
|
|
|
10
|
+
# Attach the selector for `state` after every markable cluster. Whitespace,
|
|
11
|
+
# control-break clusters, clusters ending in a provenance selector, lone selectors, and
|
|
12
|
+
# PUA provenance characters are left alone, so the operation is idempotent.
|
|
10
13
|
def _mark(text, state, mode, selectors, pua2base, base2pua)
|
|
14
|
+
raise ArgumentError, "invalid provenance state: #{state.inspect}" unless GENERATED_STATES.include?(state)
|
|
15
|
+
|
|
11
16
|
sel_cps = selectors.values.to_set
|
|
12
17
|
selector = selectors[state].chr(Encoding::UTF_8)
|
|
13
|
-
chars = text.chars
|
|
14
18
|
out = String.new(encoding: "UTF-8")
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
if sel_cps.include?(
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
start = index
|
|
25
|
-
index = cluster_end(chars, index, sel_cps)
|
|
26
|
-
cluster = chars[start...index]
|
|
27
|
-
if index < chars.length && sel_cps.include?(chars[index].ord)
|
|
28
|
-
cluster.each { |c| out << c }
|
|
29
|
-
next
|
|
30
|
-
end
|
|
31
|
-
if mode == "pua" && state == "ai" && cluster.length == 1 && base2pua.key?(cp)
|
|
32
|
-
out << base2pua[cp].chr(Encoding::UTF_8)
|
|
19
|
+
segments(text).each do |cluster|
|
|
20
|
+
first = cluster[0].ord
|
|
21
|
+
last = cluster[-1].ord
|
|
22
|
+
single = cluster.length == 1
|
|
23
|
+
if sel_cps.include?(last) || (single && pua2base.key?(first)) || blank?(cluster) ||
|
|
24
|
+
Grapheme.control_like?(last)
|
|
25
|
+
out << cluster
|
|
26
|
+
elsif mode == "pua" && state == "ai" && single && base2pua.key?(first)
|
|
27
|
+
out << base2pua[first].chr(Encoding::UTF_8)
|
|
33
28
|
else
|
|
34
|
-
|
|
35
|
-
out << selector
|
|
29
|
+
out << cluster << selector
|
|
36
30
|
end
|
|
37
31
|
end
|
|
38
32
|
out
|
|
@@ -64,23 +58,20 @@ module Textprov
|
|
|
64
58
|
return text if from_mode == to_mode
|
|
65
59
|
|
|
66
60
|
vs_ai = selectors["ai"]
|
|
67
|
-
chars = text.chars
|
|
68
61
|
out = String.new(encoding: "UTF-8")
|
|
69
62
|
if from_mode == "vs"
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
out << base2pua[char.ord].chr(Encoding::UTF_8)
|
|
76
|
-
index += 1
|
|
63
|
+
# Per cluster, not per code point: a base that shares its cluster with
|
|
64
|
+
# anything but its own selector has no PUA counterpart.
|
|
65
|
+
segments(text).each do |cluster|
|
|
66
|
+
if cluster.length == 2 && cluster[1].ord == vs_ai && base2pua.key?(cluster[0].ord)
|
|
67
|
+
out << base2pua[cluster[0].ord].chr(Encoding::UTF_8)
|
|
77
68
|
else
|
|
78
|
-
out <<
|
|
69
|
+
out << cluster
|
|
79
70
|
end
|
|
80
71
|
end
|
|
81
72
|
return out
|
|
82
73
|
end
|
|
83
|
-
|
|
74
|
+
text.each_char do |char|
|
|
84
75
|
entry = pua2base[char.ord]
|
|
85
76
|
if entry && entry[1] == "ai"
|
|
86
77
|
out << entry[0].chr(Encoding::UTF_8)
|
|
@@ -95,50 +86,41 @@ module Textprov
|
|
|
95
86
|
def _inspect(text, selectors, pua2base)
|
|
96
87
|
sel2name = selectors.each_with_object({}) { |(name, cp), h| h[cp] = name }
|
|
97
88
|
counts = { "unmarked" => 0, "human" => 0, "ai_vs" => 0, "ai_pua" => 0,
|
|
98
|
-
"
|
|
89
|
+
"whitespace" => 0 }
|
|
99
90
|
unrecognised_selectors = []
|
|
100
91
|
unrecognised_pua = []
|
|
101
|
-
sel_cp_set = selectors.values.to_set
|
|
102
92
|
|
|
103
93
|
chars = text.chars
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
94
|
+
segments(text).each do |cluster|
|
|
95
|
+
cp = cluster[0].ord
|
|
96
|
+
last = cluster[-1].ord
|
|
97
|
+
if cluster.length == 1
|
|
98
|
+
if sel2name.key?(cp) || cp.between?(0xE0100, 0xE01EF)
|
|
99
|
+
unrecognised_selectors << cp # a selector with no base in front of it
|
|
100
|
+
elsif pua2base.key?(cp)
|
|
101
|
+
counts["ai_pua"] += 1
|
|
102
|
+
elsif cp.between?(0x100000, 0x10FFFD)
|
|
103
|
+
unrecognised_pua << cp
|
|
104
|
+
elsif blank?(cluster)
|
|
105
|
+
counts["whitespace"] += 1
|
|
106
|
+
else
|
|
107
|
+
counts["unmarked"] += 1
|
|
108
|
+
end
|
|
109
|
+
elsif sel2name.key?(last) && blank?(cluster[0...-1])
|
|
110
|
+
counts["whitespace"] += 1 # whitespace cannot carry a mark
|
|
111
|
+
unrecognised_selectors << last
|
|
112
|
+
elsif sel2name.key?(last)
|
|
113
|
+
name = sel2name[last]
|
|
114
|
+
counts[name == "ai" ? "ai_vs" : name] += 1
|
|
115
|
+
elsif blank?(cluster)
|
|
126
116
|
counts["whitespace"] += 1
|
|
127
|
-
next
|
|
128
|
-
end
|
|
129
|
-
index = cluster_end(chars, index - 1, sel_cp_set)
|
|
130
|
-
if index < chars.length && sel_cp_set.include?(chars[index].ord)
|
|
131
|
-
name = sel2name[chars[index].ord]
|
|
132
|
-
index += 1
|
|
133
|
-
key = name == "ai" ? "ai_vs" : name
|
|
134
|
-
counts[key] += 1
|
|
135
117
|
else
|
|
136
118
|
counts["unmarked"] += 1
|
|
137
119
|
end
|
|
138
120
|
end
|
|
139
121
|
|
|
140
122
|
lines = ["characters: #{chars.length}"]
|
|
141
|
-
%w[unmarked human ai_vs ai_pua
|
|
123
|
+
%w[unmarked human ai_vs ai_pua whitespace].each do |key|
|
|
142
124
|
lines << "#{key}: #{counts[key]}"
|
|
143
125
|
end
|
|
144
126
|
sels = unrecognised_selectors.uniq.sort.map { |cp| format("U+%04X", cp) }.join(" ")
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# ruby/lib/textprov/grapheme.rb
|
|
2
|
+
#
|
|
3
|
+
# frozen_string_literal: true
|
|
4
|
+
|
|
5
|
+
require_relative "ucd"
|
|
6
|
+
|
|
7
|
+
module Textprov
|
|
8
|
+
# Extended grapheme cluster segmentation (UAX #29).
|
|
9
|
+
#
|
|
10
|
+
# The segmenter reads the property tables in ucd.rb, generated from one
|
|
11
|
+
# pinned Unicode version, instead of the Unicode version Onigmo ships with
|
|
12
|
+
# the interpreter. Every port segments from the same tables, so a cluster
|
|
13
|
+
# marked by one decodes as one cluster in the others. Rule numbers below
|
|
14
|
+
# refer to UAX #29, Grapheme Cluster Boundary Rules.
|
|
15
|
+
module Grapheme
|
|
16
|
+
CR = UCD::GCB["CR"]
|
|
17
|
+
LF = UCD::GCB["LF"]
|
|
18
|
+
CONTROL = UCD::GCB["Control"]
|
|
19
|
+
EXTEND = UCD::GCB["Extend"]
|
|
20
|
+
ZWJ = UCD::GCB["ZWJ"]
|
|
21
|
+
RI = UCD::GCB["Regional_Indicator"]
|
|
22
|
+
PREPEND = UCD::GCB["Prepend"]
|
|
23
|
+
SPACING_MARK = UCD::GCB["SpacingMark"]
|
|
24
|
+
L = UCD::GCB["L"]
|
|
25
|
+
V = UCD::GCB["V"]
|
|
26
|
+
T = UCD::GCB["T"]
|
|
27
|
+
LV = UCD::GCB["LV"]
|
|
28
|
+
LVT = UCD::GCB["LVT"]
|
|
29
|
+
CONTROL_LIKE = [CONTROL, CR, LF].freeze
|
|
30
|
+
AFTER_L = [L, V, LV, LVT].freeze
|
|
31
|
+
BEFORE_V = [LV, V].freeze
|
|
32
|
+
V_OR_T = [V, T].freeze
|
|
33
|
+
BEFORE_T = [LVT, T].freeze
|
|
34
|
+
EXTENDING = [EXTEND, ZWJ, SPACING_MARK].freeze
|
|
35
|
+
CONSONANT = UCD::INCB["Consonant"]
|
|
36
|
+
LINKER = UCD::INCB["Linker"]
|
|
37
|
+
INCB_EXTEND = UCD::INCB["Extend"]
|
|
38
|
+
EXTENDED_PICTOGRAPHIC = UCD::EXTENDED_PICTOGRAPHIC
|
|
39
|
+
GCB_MASK = 15
|
|
40
|
+
INCB_MASK = 224
|
|
41
|
+
|
|
42
|
+
# The look-behind a boundary decision needs beyond the previous code
|
|
43
|
+
# point: how many regional indicators precede it, whether an emoji ZWJ
|
|
44
|
+
# sequence is open (GB11), and whether an Indic conjunct is open (GB9c).
|
|
45
|
+
State = Struct.new(:regional_run, :pictograph, :conjunct)
|
|
46
|
+
|
|
47
|
+
module_function
|
|
48
|
+
|
|
49
|
+
# The property byte for one code point.
|
|
50
|
+
def property(codepoint)
|
|
51
|
+
index = UCD::BOUNDS.bsearch_index { |bound| bound > codepoint }
|
|
52
|
+
UCD::CODES[index ? index - 1 : -1]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# GB4 forces a break after these code points, even before Extend.
|
|
56
|
+
def control_like?(codepoint)
|
|
57
|
+
CONTROL_LIKE.include?(property(codepoint) & GCB_MASK)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Split `text` into extended grapheme clusters.
|
|
61
|
+
def segments(text)
|
|
62
|
+
clusters = []
|
|
63
|
+
return clusters if text.empty?
|
|
64
|
+
|
|
65
|
+
chars = text.chars
|
|
66
|
+
current = chars[0].dup
|
|
67
|
+
previous = property(chars[0].ord)
|
|
68
|
+
state = advance(State.new(0, 0, 0), previous)
|
|
69
|
+
chars[1..].each do |character|
|
|
70
|
+
following = property(character.ord)
|
|
71
|
+
if join?(previous, following, state)
|
|
72
|
+
current << character
|
|
73
|
+
else
|
|
74
|
+
clusters << current
|
|
75
|
+
current = character.dup
|
|
76
|
+
end
|
|
77
|
+
state = advance(state, following)
|
|
78
|
+
previous = following
|
|
79
|
+
end
|
|
80
|
+
clusters << current
|
|
81
|
+
clusters
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Is there no boundary between a code point with property byte `previous`
|
|
85
|
+
# and one with `following`? One guard per rule, in rule order.
|
|
86
|
+
def join?(previous, following, state) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
87
|
+
before = previous & GCB_MASK
|
|
88
|
+
after = following & GCB_MASK
|
|
89
|
+
return true if before == CR && after == LF # GB3
|
|
90
|
+
return false if CONTROL_LIKE.include?(before) || CONTROL_LIKE.include?(after) # GB4, GB5
|
|
91
|
+
return true if before == L && AFTER_L.include?(after) # GB6
|
|
92
|
+
return true if BEFORE_V.include?(before) && V_OR_T.include?(after) # GB7
|
|
93
|
+
return true if BEFORE_T.include?(before) && after == T # GB8
|
|
94
|
+
return true if EXTENDING.include?(after) # GB9, GB9a
|
|
95
|
+
return true if before == PREPEND # GB9b
|
|
96
|
+
return true if state.conjunct == 2 && (following & INCB_MASK) == CONSONANT # GB9c
|
|
97
|
+
return true if state.pictograph == 2 && following.anybits?(EXTENDED_PICTOGRAPHIC) # GB11
|
|
98
|
+
return state.regional_run.odd? if before == RI && after == RI # GB12, GB13
|
|
99
|
+
|
|
100
|
+
false # GB999
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The state after consuming a code point with property byte `following`.
|
|
104
|
+
def advance(state, following)
|
|
105
|
+
gcb = following & GCB_MASK
|
|
106
|
+
regional_run = gcb == RI ? state.regional_run + 1 : 0
|
|
107
|
+
# 0: nothing; 1: Extended_Pictographic Extend*; 2: that followed by ZWJ.
|
|
108
|
+
open = state.pictograph == 1
|
|
109
|
+
pictograph =
|
|
110
|
+
if following.anybits?(EXTENDED_PICTOGRAPHIC) || (open && gcb == EXTEND) then 1
|
|
111
|
+
elsif open && gcb == ZWJ then 2
|
|
112
|
+
else 0
|
|
113
|
+
end
|
|
114
|
+
# 0: nothing; 1: Consonant (Extend|Linker)* without a Linker yet;
|
|
115
|
+
# 2: Consonant ... Linker (Extend|Linker)*.
|
|
116
|
+
incb = following & INCB_MASK
|
|
117
|
+
conjunct = conjunct_after(state.conjunct, incb)
|
|
118
|
+
State.new(regional_run, pictograph, conjunct)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def conjunct_after(conjunct, incb)
|
|
122
|
+
return 1 if incb == CONSONANT
|
|
123
|
+
return 0 if conjunct.zero?
|
|
124
|
+
return 2 if incb == LINKER
|
|
125
|
+
|
|
126
|
+
incb == INCB_EXTEND ? conjunct : 0
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
data/lib/textprov/mapping.json
CHANGED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/* js/textprov.css */
|
|
2
|
+
|
|
3
|
+
/*
|
|
4
|
+
* Optional styles for the TextProv baseline demonstration.
|
|
5
|
+
* Classifications are human and ai; these styles do not implement voice references.
|
|
6
|
+
* Mapping: ../experiments/baseline/mapping.json (experimental version 0.2).
|
|
7
|
+
* Applications may choose other presentation without changing the stored text.
|
|
8
|
+
* The reveal treatment is gradient mist with an 80%-strength dotted underline.
|
|
9
|
+
* The former wavy/sawtooth provenance underline is obsolete.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/* Native application annotations remain above the optional reveal layer. */
|
|
13
|
+
@layer textprov {
|
|
14
|
+
|
|
15
|
+
:root {
|
|
16
|
+
--prov-human-bg: rgba(46, 160, 67, 0.14);
|
|
17
|
+
--prov-human-line: #2ea043;
|
|
18
|
+
--prov-ai-bg: rgba(199, 125, 0, 0.14);
|
|
19
|
+
--prov-ai-line: #c77d00;
|
|
20
|
+
--prov-mixed-line: #6e7781;
|
|
21
|
+
--prov-mixed-bg: rgba(110, 119, 129, 0.14);
|
|
22
|
+
--prov-tint: 14%;
|
|
23
|
+
--prov-dot-strength: 80%;
|
|
24
|
+
--prov-glow-opacity: 27%;
|
|
25
|
+
--prov-glow-blur: 11.2px;
|
|
26
|
+
--prov-glow-spread: 4px;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
@media (prefers-color-scheme: dark) {
|
|
30
|
+
:root {
|
|
31
|
+
--prov-human-bg: rgba(63, 185, 80, 0.16);
|
|
32
|
+
--prov-human-line: #3fb950;
|
|
33
|
+
--prov-ai-bg: rgba(227, 152, 33, 0.16);
|
|
34
|
+
--prov-ai-line: #e39821;
|
|
35
|
+
--prov-mixed-line: #a0a7b0;
|
|
36
|
+
--prov-mixed-bg: rgba(160, 167, 176, 0.16);
|
|
37
|
+
--prov-glow-opacity: 17.55%;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.prov,
|
|
42
|
+
.prov-human,
|
|
43
|
+
.prov-ai,
|
|
44
|
+
[data-prov="human"],
|
|
45
|
+
[data-prov="ai"],
|
|
46
|
+
[data-prov-passage] {
|
|
47
|
+
/* No border/padding/margin/line-height: marks must not affect layout. */
|
|
48
|
+
box-decoration-break: clone;
|
|
49
|
+
-webkit-box-decoration-break: clone;
|
|
50
|
+
border-radius: 3px;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
.prov-human,
|
|
54
|
+
[data-prov="human"],
|
|
55
|
+
[data-prov-passage="human"] {
|
|
56
|
+
--prov-color: var(--prov-human-line);
|
|
57
|
+
--prov-fallback-bg: var(--prov-human-bg);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.prov-ai,
|
|
61
|
+
[data-prov="ai"],
|
|
62
|
+
[data-prov-passage="ai"] {
|
|
63
|
+
--prov-color: var(--prov-ai-line);
|
|
64
|
+
--prov-fallback-bg: var(--prov-ai-bg);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
.prov-human,
|
|
68
|
+
.prov-ai,
|
|
69
|
+
[data-prov="human"],
|
|
70
|
+
[data-prov="ai"],
|
|
71
|
+
[data-prov-passage] {
|
|
72
|
+
/* The solid tint is a fallback for engines without color-mix(). */
|
|
73
|
+
background: var(--prov-fallback-bg);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
:is(.prov-human, .prov-ai, [data-prov="human"], [data-prov="ai"], [data-prov-passage]):not(a) {
|
|
77
|
+
text-decoration-line: underline;
|
|
78
|
+
text-decoration-style: dotted;
|
|
79
|
+
text-decoration-color: var(--prov-color);
|
|
80
|
+
text-decoration-thickness: 1.5px;
|
|
81
|
+
text-decoration-skip-ink: none;
|
|
82
|
+
text-underline-offset: 5px;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
@supports (background: color-mix(in srgb, currentColor 50%, transparent)) {
|
|
86
|
+
.prov-human,
|
|
87
|
+
.prov-ai,
|
|
88
|
+
[data-prov="human"],
|
|
89
|
+
[data-prov="ai"],
|
|
90
|
+
[data-prov-passage] {
|
|
91
|
+
background: linear-gradient(
|
|
92
|
+
115deg,
|
|
93
|
+
color-mix(in srgb, var(--prov-color) calc(var(--prov-tint) * 0.45), transparent),
|
|
94
|
+
color-mix(in srgb, var(--prov-color) calc(var(--prov-tint) * 1.6), transparent) 55%,
|
|
95
|
+
color-mix(in srgb, var(--prov-color) calc(var(--prov-tint) * 0.6), transparent)
|
|
96
|
+
);
|
|
97
|
+
box-shadow: 0 0 var(--prov-glow-blur) var(--prov-glow-spread)
|
|
98
|
+
color-mix(in srgb, var(--prov-color) var(--prov-glow-opacity), transparent);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
:is(.prov-human, .prov-ai, [data-prov="human"], [data-prov="ai"], [data-prov-passage]):not(a) {
|
|
102
|
+
text-decoration-color: color-mix(in srgb, var(--prov-color) var(--prov-dot-strength), transparent);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/* A group describes a passage containing marks, not every character's origin. */
|
|
107
|
+
[data-prov-passage="mixed"] {
|
|
108
|
+
--prov-color: var(--prov-mixed-line);
|
|
109
|
+
--prov-fallback-bg: var(--prov-mixed-bg);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/* Exact spans always remain in the DOM. Only the chosen display layer paints. */
|
|
113
|
+
[data-prov-passage] [data-prov]:not([data-prov-display="exact"] *),
|
|
114
|
+
[data-prov-display="exact"] [data-prov-passage] {
|
|
115
|
+
background: none;
|
|
116
|
+
box-shadow: none;
|
|
117
|
+
text-decoration: none;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/* Existing link underlines keep their channel. For spelling/grammar cues,
|
|
121
|
+
* put data-prov-decoration="native" on the enclosing proofing range. */
|
|
122
|
+
a :is(.prov, [data-prov="human"], [data-prov="ai"], [data-prov-passage]),
|
|
123
|
+
[data-prov-decoration="native"] :is(.prov, [data-prov="human"], [data-prov="ai"], [data-prov-passage]):not(a),
|
|
124
|
+
:is(.prov, [data-prov="human"], [data-prov="ai"], [data-prov-passage])[data-prov-decoration="native"]:not(a) {
|
|
125
|
+
text-decoration: none;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
[data-prov-passage]:has([data-prov-decoration="native"]) { text-decoration: none; }
|
|
129
|
+
|
|
130
|
+
/* Comment anchors and other native fills can retain their own background. */
|
|
131
|
+
[data-prov-background="native"] :is(.prov, [data-prov="human"], [data-prov="ai"], [data-prov-passage]),
|
|
132
|
+
:is(.prov, [data-prov="human"], [data-prov="ai"], [data-prov-passage])[data-prov-background="native"] {
|
|
133
|
+
background: none;
|
|
134
|
+
box-shadow: none;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
[data-prov-passage]:has([data-prov-background="native"]) { background: none; box-shadow: none; }
|
|
138
|
+
|
|
139
|
+
@media (forced-colors: active) {
|
|
140
|
+
.prov-human,
|
|
141
|
+
.prov-ai,
|
|
142
|
+
[data-prov="human"],
|
|
143
|
+
[data-prov="ai"],
|
|
144
|
+
[data-prov-passage] {
|
|
145
|
+
background: none;
|
|
146
|
+
box-shadow: none;
|
|
147
|
+
text-decoration-color: currentColor;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
}
|