polytypo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +109 -0
- data/lib/polytypo/data/README.md +20 -0
- data/lib/polytypo/data/UNICODE +1 -0
- data/lib/polytypo/data/VERSION +1 -0
- data/lib/polytypo/data/fixtures/de-CH.json +501 -0
- data/lib/polytypo/data/fixtures/de-DE.json +547 -0
- data/lib/polytypo/data/fixtures/el.json +239 -0
- data/lib/polytypo/data/fixtures/en-GB.json +1274 -0
- data/lib/polytypo/data/fixtures/en-US.json +1807 -0
- data/lib/polytypo/data/fixtures/fi.json +1306 -0
- data/lib/polytypo/data/fixtures/fr-CA.json +268 -0
- data/lib/polytypo/data/fixtures/fr.json +603 -0
- data/lib/polytypo/data/fixtures/locale-resolution.json +209 -0
- data/lib/polytypo/data/fixtures/ru.json +688 -0
- data/lib/polytypo/data/fixtures/sv.json +1290 -0
- data/lib/polytypo/data/locales/de-CH.json +77 -0
- data/lib/polytypo/data/locales/de-DE.json +76 -0
- data/lib/polytypo/data/locales/el.json +90 -0
- data/lib/polytypo/data/locales/en-GB.json +115 -0
- data/lib/polytypo/data/locales/en-US.json +133 -0
- data/lib/polytypo/data/locales/fi.json +136 -0
- data/lib/polytypo/data/locales/fr-CA.json +78 -0
- data/lib/polytypo/data/locales/fr.json +84 -0
- data/lib/polytypo/data/locales/registry.json +9 -0
- data/lib/polytypo/data/locales/ru.json +112 -0
- data/lib/polytypo/data/locales/sv.json +124 -0
- data/lib/polytypo/data/rules/dashes.md +1238 -0
- data/lib/polytypo/data/rules/order.json +78 -0
- data/lib/polytypo/data/schema/fixtures.schema.json +79 -0
- data/lib/polytypo/data/schema/locale.schema.json +235 -0
- data/lib/polytypo/data/schema/registry.schema.json +29 -0
- data/lib/polytypo/data/schema/resolution.schema.json +50 -0
- data/lib/polytypo/engine/codepoints.rb +24 -0
- data/lib/polytypo/engine/edits.rb +64 -0
- data/lib/polytypo/engine/locale.rb +138 -0
- data/lib/polytypo/engine/pipeline.rb +61 -0
- data/lib/polytypo/engine/registry.rb +47 -0
- data/lib/polytypo/engine/rules/apostrophe.rb +127 -0
- data/lib/polytypo/engine/rules/dash_shared.rb +342 -0
- data/lib/polytypo/engine/rules/dashes.rb +125 -0
- data/lib/polytypo/engine/rules/ellipsis.rb +100 -0
- data/lib/polytypo/engine/rules/hyphen.rb +207 -0
- data/lib/polytypo/engine/rules/nbsp.rb +616 -0
- data/lib/polytypo/engine/rules/quote_ambiguity.rb +241 -0
- data/lib/polytypo/engine/rules/quotes.rb +420 -0
- data/lib/polytypo/engine/rules/ranges.rb +124 -0
- data/lib/polytypo/engine/rules/spaces.rb +232 -0
- data/lib/polytypo/engine/rules/symbols.rb +291 -0
- data/lib/polytypo/engine/rules.rb +19 -0
- data/lib/polytypo/engine/sentinels.rb +23 -0
- data/lib/polytypo/engine/unicode_util.rb +390 -0
- data/lib/polytypo/errors.rb +24 -0
- data/lib/polytypo/modes/html.rb +233 -0
- data/lib/polytypo/modes/markdown.rb +187 -0
- data/lib/polytypo/modes/parse_error.rb +19 -0
- data/lib/polytypo/modes/runner.rb +57 -0
- data/lib/polytypo/modes/spans.rb +132 -0
- data/lib/polytypo/version.rb +5 -0
- data/lib/polytypo.rb +91 -0
- data/polytypo.gemspec +37 -0
- metadata +122 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "spans"
|
|
4
|
+
require_relative "html"
|
|
5
|
+
require_relative "parse_error"
|
|
6
|
+
require_relative "../errors"
|
|
7
|
+
|
|
8
|
+
module Polytypo
|
|
9
|
+
module Modes
|
|
10
|
+
# Markdown span extraction via `commonmarker` (the Ruby binding to the `comrak` Rust
|
|
11
|
+
# CommonMark/GFM engine). `dialect="mdx"` returns POLYTYPO_INVALID_DIALECT: this runtime has
|
|
12
|
+
# no MDX/JSX parser, which the spec permits (a narrower conformance claim, not a silent
|
|
13
|
+
# mishandling of a dialect it claims to support).
|
|
14
|
+
#
|
|
15
|
+
# `require "commonmarker"` happens lazily, inside this file only, never at the top of
|
|
16
|
+
# lib/polytypo.rb -- a caller who only ever uses mode: "text" never loads the native
|
|
17
|
+
# commonmarker extension.
|
|
18
|
+
module Markdown
|
|
19
|
+
def self.resolve_dialect(dialect)
|
|
20
|
+
case dialect
|
|
21
|
+
when nil, ""
|
|
22
|
+
raise Polytypo::Error.new(
|
|
23
|
+
Polytypo::CODE_INVALID_DIALECT,
|
|
24
|
+
'mode "markdown" requires a dialect; there is no default'
|
|
25
|
+
)
|
|
26
|
+
when "commonmark"
|
|
27
|
+
nil
|
|
28
|
+
when "mdx"
|
|
29
|
+
raise Polytypo::Error.new(
|
|
30
|
+
Polytypo::CODE_INVALID_DIALECT,
|
|
31
|
+
'dialect "mdx" is not supported by this runtime (no MDX/JSX parser available); ' \
|
|
32
|
+
'only "commonmark" is supported'
|
|
33
|
+
)
|
|
34
|
+
else
|
|
35
|
+
raise Polytypo::Error.new(
|
|
36
|
+
Polytypo::CODE_INVALID_DIALECT,
|
|
37
|
+
"unknown dialect #{dialect.inspect}; this runtime supports \"commonmark\""
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Block/inline node types that are never processable, in full (modes.md 3.7.3): fenced and
|
|
43
|
+
# indented code (comrak represents both as :code_block), inline code spans, thematic
|
|
44
|
+
# breaks (no prose content), and frontmatter (skipped whole including its delimiters).
|
|
45
|
+
SKIP_WHOLE_TYPES = %i[code code_block thematic_break frontmatter].freeze
|
|
46
|
+
|
|
47
|
+
# A leaf whose Segment/source_position directly delineates literal, processable prose --
|
|
48
|
+
# comrak's AST is segment-based like goldmark's: structural bytes (emphasis delimiters,
|
|
49
|
+
# link brackets/destination, table pipes and the delimiter row, list markers, blockquote
|
|
50
|
+
# markers, heading hashes, fence lines and info strings) are consumed during parsing and
|
|
51
|
+
# never represented as a node's own source_position at all, so no gap-filling step is
|
|
52
|
+
# needed here -- emitting a span for exactly every :text node's source_position already
|
|
53
|
+
# reconstructs precisely the processable prose, nothing more (same discovery as the Go
|
|
54
|
+
# port's goldmark-based adapter).
|
|
55
|
+
TEXT_LEAF_TYPES = %i[text].freeze
|
|
56
|
+
|
|
57
|
+
# detect_frontmatter_delimiter: comrak's frontmatter extension takes one literal delimiter
|
|
58
|
+
# string and requires the opening AND closing lines to match it exactly -- unlike the Go
|
|
59
|
+
# port (which had to hand-roll frontmatter detection entirely), here only the *which
|
|
60
|
+
# delimiter* choice needs to be made upfront, from the document's own leading bytes.
|
|
61
|
+
def self.detect_frontmatter_delimiter(source)
|
|
62
|
+
return "+++" if source.start_with?("+++")
|
|
63
|
+
return "---" if source.start_with?("---")
|
|
64
|
+
|
|
65
|
+
nil
|
|
66
|
+
end
|
|
67
|
+
private_class_method :detect_frontmatter_delimiter
|
|
68
|
+
|
|
69
|
+
# Locates the processable spans of a Markdown document. dialect must already be validated
|
|
70
|
+
# via resolve_dialect (== "commonmark"); this function does not re-check it.
|
|
71
|
+
def self.markdown_spans(source)
|
|
72
|
+
require "commonmarker"
|
|
73
|
+
|
|
74
|
+
cp = Polytypo::Engine::Codepoints.to_codepoints(source)
|
|
75
|
+
chars = cp.map { |c| [c].pack("U") }
|
|
76
|
+
line_starts = build_line_starts(chars)
|
|
77
|
+
|
|
78
|
+
delimiter = detect_frontmatter_delimiter(source)
|
|
79
|
+
options = { parse: { sourcepos_chars: true } }
|
|
80
|
+
options[:extension] = { front_matter_delimiter: delimiter } if delimiter
|
|
81
|
+
|
|
82
|
+
spans = []
|
|
83
|
+
ParseError.wrap do
|
|
84
|
+
doc = Commonmarker.parse(source, options: options)
|
|
85
|
+
html_stack_holder = { stack: [] }
|
|
86
|
+
walk(doc, chars, line_starts, spans, html_stack_holder)
|
|
87
|
+
end
|
|
88
|
+
spans
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# line_starts[i] is the 0-based code-point index at which comrak's 1-based line i begins.
|
|
92
|
+
def self.build_line_starts(chars)
|
|
93
|
+
starts = [0]
|
|
94
|
+
chars.each_with_index do |ch, i|
|
|
95
|
+
starts << (i + 1) if ch == "\n"
|
|
96
|
+
end
|
|
97
|
+
starts
|
|
98
|
+
end
|
|
99
|
+
private_class_method :build_line_starts
|
|
100
|
+
|
|
101
|
+
def self.offset_of(line_starts, line, column)
|
|
102
|
+
line_starts[line - 1] + (column - 1)
|
|
103
|
+
end
|
|
104
|
+
private_class_method :offset_of
|
|
105
|
+
|
|
106
|
+
def self.walk(node, chars, line_starts, spans, html_holder)
|
|
107
|
+
type = node.type
|
|
108
|
+
return if SKIP_WHOLE_TYPES.include?(type)
|
|
109
|
+
|
|
110
|
+
if type == :html_block
|
|
111
|
+
handle_html_block(node, chars, line_starts, spans)
|
|
112
|
+
return
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
if type == :html_inline
|
|
116
|
+
handle_html_inline(node, chars, line_starts, html_holder)
|
|
117
|
+
return
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
if type == :link || type == :image
|
|
121
|
+
sp = node.source_position
|
|
122
|
+
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
123
|
+
unless chars[start_offset] == "["
|
|
124
|
+
# No leading "[" at the link's own source position: this is an autolink
|
|
125
|
+
# (<https://…> or a bare GFM autolink literal), not a real [text](url) link --
|
|
126
|
+
# skipped whole, per modes.md 3.7.3.
|
|
127
|
+
return
|
|
128
|
+
end
|
|
129
|
+
# A real link/image: fall through to the generic child walk below, which emits the
|
|
130
|
+
# link text (or image alt text) as processable prose. The destination/title are never
|
|
131
|
+
# represented as child nodes at all, so there is nothing further to exclude.
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
if TEXT_LEAF_TYPES.include?(type)
|
|
135
|
+
sp = node.source_position
|
|
136
|
+
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
137
|
+
end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1
|
|
138
|
+
if html_holder[:stack].empty? && end_offset > start_offset
|
|
139
|
+
spans << Spans::Span.new(start_offset, end_offset)
|
|
140
|
+
end
|
|
141
|
+
return
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
if %i[paragraph heading table_cell].include?(type)
|
|
145
|
+
# A fresh top-level inline-bearing container: the raw-HTML skip stack resets here (an
|
|
146
|
+
# unclosed skipped start tag skips to the end of the block, modes.md 3.7.3).
|
|
147
|
+
html_holder[:stack] = []
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
node.each { |child| walk(child, chars, line_starts, spans, html_holder) }
|
|
151
|
+
end
|
|
152
|
+
private_class_method :walk
|
|
153
|
+
|
|
154
|
+
def self.handle_html_block(node, chars, line_starts, spans)
|
|
155
|
+
sp = node.source_position
|
|
156
|
+
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
157
|
+
end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1
|
|
158
|
+
return if end_offset <= start_offset
|
|
159
|
+
|
|
160
|
+
text = chars[start_offset...end_offset].join
|
|
161
|
+
Html.html_spans(text).each do |s|
|
|
162
|
+
spans << Spans::Span.new(start_offset + s.start, start_offset + s.end)
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
private_class_method :handle_html_block
|
|
166
|
+
|
|
167
|
+
def self.handle_html_inline(node, chars, line_starts, html_holder)
|
|
168
|
+
sp = node.source_position
|
|
169
|
+
start_offset = offset_of(line_starts, sp[:start_line], sp[:start_column])
|
|
170
|
+
end_offset = offset_of(line_starts, sp[:end_line], sp[:end_column]) + 1
|
|
171
|
+
raw = chars[start_offset...end_offset]
|
|
172
|
+
name, closing, self_closing = Html.read_tag(raw) || [nil, nil, nil]
|
|
173
|
+
return if name.nil?
|
|
174
|
+
|
|
175
|
+
stack = html_holder[:stack]
|
|
176
|
+
if closing
|
|
177
|
+
if !stack.empty? && stack[-1] == name
|
|
178
|
+
stack.pop
|
|
179
|
+
end
|
|
180
|
+
elsif !self_closing && Html::SKIPPED_ELEMENTS.include?(name)
|
|
181
|
+
stack << name
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
private_class_method :handle_html_inline
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../errors"
|
|
4
|
+
|
|
5
|
+
module Polytypo
|
|
6
|
+
module Modes
|
|
7
|
+
# Wraps any error either mode parser produces into POLYTYPO_MALFORMED_INPUT -- no third-party
|
|
8
|
+
# parser's error type is ever allowed to escape Transform.
|
|
9
|
+
module ParseError
|
|
10
|
+
def self.wrap
|
|
11
|
+
yield
|
|
12
|
+
rescue Polytypo::Error
|
|
13
|
+
raise
|
|
14
|
+
rescue StandardError => e
|
|
15
|
+
raise Polytypo::Error.new(Polytypo::CODE_MALFORMED_INPUT, "input did not parse: #{e.message}")
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "spans"
|
|
4
|
+
require_relative "../engine/edits"
|
|
5
|
+
require_relative "../engine/registry"
|
|
6
|
+
|
|
7
|
+
module Polytypo
|
|
8
|
+
module Modes
|
|
9
|
+
# modes.md 3.5. The pipeline runs once, over the marker-separated concatenation of every
|
|
10
|
+
# processable span -- not per span (would pair quotation marks in isolation), and not over a
|
|
11
|
+
# naive concatenation (would manufacture adjacencies the document does not have).
|
|
12
|
+
module Runner
|
|
13
|
+
# The same sequence as Engine::Pipeline.run_rules, with the two boundary filters of
|
|
14
|
+
# modes.md 3.4 interposed.
|
|
15
|
+
def self.run_rules_over_spans(cp, plan, locale_data, ctx)
|
|
16
|
+
current = cp
|
|
17
|
+
plan.each do |rule_id|
|
|
18
|
+
fn = Engine::Registry.rule(rule_id)
|
|
19
|
+
edits = fn.call(current, locale_data, ctx)
|
|
20
|
+
filtered = Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current))
|
|
21
|
+
next if filtered.empty?
|
|
22
|
+
|
|
23
|
+
current = Engine::Edits.apply_edits(current, filtered, rule_id)
|
|
24
|
+
end
|
|
25
|
+
current
|
|
26
|
+
end
|
|
27
|
+
private_class_method :run_rules_over_spans
|
|
28
|
+
|
|
29
|
+
# The output is the input with a set of disjoint substring replacements applied and nothing
|
|
30
|
+
# else (modes.md 4). A span whose content the rules did not change contributes no
|
|
31
|
+
# replacement, so a document needing no changes comes back byte-identical.
|
|
32
|
+
#
|
|
33
|
+
# source_cp is the whole input already converted to code points; spans address it by
|
|
34
|
+
# code-point index.
|
|
35
|
+
def self.run_over_spans(source_cp, spans, plan, locale_data, ctx)
|
|
36
|
+
normalized = Spans.normalize_spans(spans)
|
|
37
|
+
return source_cp.pack("U*") if normalized.empty?
|
|
38
|
+
|
|
39
|
+
concatenated = Spans.concatenate_spans(source_cp, normalized)
|
|
40
|
+
transformed = run_rules_over_spans(concatenated, plan, locale_data, ctx)
|
|
41
|
+
pieces = Spans.split_on_marker(transformed, normalized.length)
|
|
42
|
+
|
|
43
|
+
out = []
|
|
44
|
+
cursor = 0
|
|
45
|
+
normalized.each_with_index do |span, i|
|
|
46
|
+
piece = pieces[i]
|
|
47
|
+
original = source_cp[span.start...span.end]
|
|
48
|
+
out.concat(source_cp[cursor...span.start])
|
|
49
|
+
out.concat(piece == original ? original : piece)
|
|
50
|
+
cursor = span.end
|
|
51
|
+
end
|
|
52
|
+
out.concat(source_cp[cursor..])
|
|
53
|
+
out.pack("U*")
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../engine/sentinels"
|
|
4
|
+
require_relative "../errors"
|
|
5
|
+
|
|
6
|
+
module Polytypo
|
|
7
|
+
module Modes
|
|
8
|
+
# The span model shared by every mode adapter that is not "text" (spec/rules/modes.md
|
|
9
|
+
# 3.2-3.5). Mirrors the other three ports' spans.{ts,py,go} exactly: markers are written here
|
|
10
|
+
# (mode layer), classified in Engine::Sentinels and in each rule's own class definitions (L1).
|
|
11
|
+
module Spans
|
|
12
|
+
LINE_TERMINATORS = [0x0A, 0x0D, 0x0B, 0x0C, 0x85, 0x2028, 0x2029].freeze
|
|
13
|
+
|
|
14
|
+
# A processable span, identified by its offsets in the original source, addressed as
|
|
15
|
+
# code-point indices (ARCHITECTURE.md section 4.2) -- Ruby String indices are already
|
|
16
|
+
# code-point indices for a UTF-8 String, no separate byte/char distinction to manage here.
|
|
17
|
+
Span = Struct.new(:start, :end)
|
|
18
|
+
|
|
19
|
+
# A span's extent in the concatenated code-point array: s0/s1 of modes.md 3.4. `:first`
|
|
20
|
+
# deliberately shadows Struct#first with the equivalent one-argument-less accessor -- this
|
|
21
|
+
# struct is never called with an integer argument the way Struct#first(n) would expect, and
|
|
22
|
+
# `first`/`last` match every other port's SpanRange naming exactly.
|
|
23
|
+
SpanRange = Struct.new(:first, :last) # rubocop:disable Lint/StructNewOverride
|
|
24
|
+
|
|
25
|
+
def self.gap_is_line_boundary?(cp, from, to)
|
|
26
|
+
(from...to).any? { |i| LINE_TERMINATORS.include?(cp[i]) }
|
|
27
|
+
end
|
|
28
|
+
private_class_method :gap_is_line_boundary?
|
|
29
|
+
|
|
30
|
+
# Sorts, drops empties, and coalesces spans separated by nothing in the source (modes.md
|
|
31
|
+
# 7.5). Overlapping spans are an extractor bug and are rejected rather than silently merged.
|
|
32
|
+
def self.normalize_spans(spans)
|
|
33
|
+
sorted = spans.select { |s| s.end > s.start }.sort_by(&:start)
|
|
34
|
+
out = []
|
|
35
|
+
sorted.each do |span|
|
|
36
|
+
if out.empty?
|
|
37
|
+
out << span
|
|
38
|
+
next
|
|
39
|
+
end
|
|
40
|
+
last = out[-1]
|
|
41
|
+
if span.start < last.end
|
|
42
|
+
raise Polytypo::Error.new(
|
|
43
|
+
Polytypo::CODE_RULE_CONTRACT,
|
|
44
|
+
"mode extractor produced overlapping spans (#{last.start}, #{last.end}) and " \
|
|
45
|
+
"(#{span.start}, #{span.end})"
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
if span.start == last.end
|
|
49
|
+
out[-1] = Span.new(last.start, span.end)
|
|
50
|
+
else
|
|
51
|
+
out << span
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
out
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# S1 (marker) S2 ... Sm (modes.md 3.5 step 2), given the source's full code-point array.
|
|
58
|
+
def self.concatenate_spans(source_cp, spans)
|
|
59
|
+
cp = []
|
|
60
|
+
previous = nil
|
|
61
|
+
spans.each do |span|
|
|
62
|
+
unless previous.nil?
|
|
63
|
+
cp << (gap_is_line_boundary?(source_cp, previous.end, span.start) ? Engine::LINE_MARKER : Engine::MARKER)
|
|
64
|
+
end
|
|
65
|
+
cp.concat(source_cp[span.start...span.end])
|
|
66
|
+
previous = span
|
|
67
|
+
end
|
|
68
|
+
cp
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The span extents of the array as it stands. Recomputed after every rule, because applying
|
|
72
|
+
# edits shifts every index after the first one -- the markers themselves always survive,
|
|
73
|
+
# since no edit may contain one.
|
|
74
|
+
def self.span_ranges_of(cp)
|
|
75
|
+
ranges = []
|
|
76
|
+
first = 0
|
|
77
|
+
cp.each_with_index do |value, i|
|
|
78
|
+
if Engine.marker?(value)
|
|
79
|
+
ranges << SpanRange.new(first, i - 1)
|
|
80
|
+
first = i + 1
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
ranges << SpanRange.new(first, cp.length - 1)
|
|
84
|
+
ranges
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def self.span_containing(ranges, p)
|
|
88
|
+
ranges.find { |r| r.first <= p && p <= r.last + 1 }
|
|
89
|
+
end
|
|
90
|
+
private_class_method :span_containing
|
|
91
|
+
|
|
92
|
+
# modes.md 3.4, two safety nets, both pure functions of (p, q, r, s0, s1):
|
|
93
|
+
#
|
|
94
|
+
# 1. No edit may contain a marker -- one that does is a bug, discarded rather than
|
|
95
|
+
# redistributed.
|
|
96
|
+
# 2. The edge-growth rule: an edit is discarded if it would place code points at an
|
|
97
|
+
# extremity of its span that were not there before.
|
|
98
|
+
def self.filter_boundary_edits(cp, edits, ranges)
|
|
99
|
+
edits.select do |edit|
|
|
100
|
+
next false if (edit.start...edit.end).any? { |i| Engine.marker?(cp[i]) }
|
|
101
|
+
|
|
102
|
+
p = edit.start
|
|
103
|
+
q = edit.end - 1
|
|
104
|
+
d = edit.end - edit.start
|
|
105
|
+
r = edit.replacement.length
|
|
106
|
+
span = span_containing(ranges, p)
|
|
107
|
+
!(span && r > d && (p == span.first || q == span.last))
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Redistributes the transformed array back to one piece per span (modes.md 3.5 step 4).
|
|
112
|
+
def self.split_on_marker(cp, expected)
|
|
113
|
+
pieces = [[]]
|
|
114
|
+
cp.each do |value|
|
|
115
|
+
if Engine.marker?(value)
|
|
116
|
+
pieces << []
|
|
117
|
+
else
|
|
118
|
+
pieces[-1] << value
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
if pieces.length != expected
|
|
122
|
+
raise Polytypo::Error.new(
|
|
123
|
+
Polytypo::CODE_RULE_CONTRACT,
|
|
124
|
+
"boundary markers did not survive the pipeline: expected #{expected} spans, " \
|
|
125
|
+
"found #{pieces.length}"
|
|
126
|
+
)
|
|
127
|
+
end
|
|
128
|
+
pieces
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
data/lib/polytypo.rb
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "polytypo/version"
|
|
4
|
+
require_relative "polytypo/errors"
|
|
5
|
+
require_relative "polytypo/engine/codepoints"
|
|
6
|
+
require_relative "polytypo/engine/pipeline"
|
|
7
|
+
require_relative "polytypo/engine/rules" # side effect: registers all 9 rules
|
|
8
|
+
require_relative "polytypo/modes/spans"
|
|
9
|
+
require_relative "polytypo/modes/runner"
|
|
10
|
+
require_relative "polytypo/modes/html"
|
|
11
|
+
|
|
12
|
+
# polytypo is locale-correct microtypography: quotes, dashes, ellipses, apostrophes, symbols and
|
|
13
|
+
# no-break spaces, from a spec shared across every polytypo runtime
|
|
14
|
+
# (github.com/polytypo/polytypo). See spec/CONFORMANCE.md there for exactly what this runtime
|
|
15
|
+
# implements.
|
|
16
|
+
module Polytypo
|
|
17
|
+
# Applies polytypo's rule pipeline to +input+ and returns the result.
|
|
18
|
+
#
|
|
19
|
+
# +locale+ is required, with no default -- an unknown locale raises Polytypo::Error with
|
|
20
|
+
# CODE_UNKNOWN_LOCALE; there is never a fallback to English. +mode+ is "text" (default), "html"
|
|
21
|
+
# or "markdown". +dialect+ is required iff mode is "markdown" ("commonmark"; "mdx" is not
|
|
22
|
+
# implemented by this runtime). +rules+ is an opt-out Hash keyed by rule id.
|
|
23
|
+
#
|
|
24
|
+
# Pure: no I/O, no environment, no clock, no globals, no class-level mutable state --
|
|
25
|
+
# thread-safe and reentrant, callable concurrently from any number of Threads with no external
|
|
26
|
+
# synchronization (ARCHITECTURE.md section 7).
|
|
27
|
+
#
|
|
28
|
+
# Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily
|
|
29
|
+
# requires "commonmarker" from within Modes::Markdown, never at load time of this file.
|
|
30
|
+
def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil)
|
|
31
|
+
resolved_mode = resolve_mode(mode)
|
|
32
|
+
|
|
33
|
+
case resolved_mode
|
|
34
|
+
when "text"
|
|
35
|
+
if dialect
|
|
36
|
+
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
37
|
+
end
|
|
38
|
+
transform_text(input, locale, rules)
|
|
39
|
+
when "html"
|
|
40
|
+
if dialect
|
|
41
|
+
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
|
|
42
|
+
end
|
|
43
|
+
transform_html(input, locale, rules)
|
|
44
|
+
else # "markdown"
|
|
45
|
+
transform_markdown(input, locale, dialect, rules)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.resolve_mode(mode)
|
|
50
|
+
case mode
|
|
51
|
+
when nil, "text"
|
|
52
|
+
"text"
|
|
53
|
+
when "html", "markdown"
|
|
54
|
+
mode
|
|
55
|
+
else
|
|
56
|
+
raise Error.new(CODE_INVALID_MODE, "unknown mode #{mode.inspect}. Expected \"text\", \"html\" or \"markdown\"")
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
private_class_method :resolve_mode
|
|
60
|
+
|
|
61
|
+
def self.transform_text(input, locale, rules)
|
|
62
|
+
_resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
63
|
+
cp = Engine::Codepoints.to_codepoints(input)
|
|
64
|
+
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale)
|
|
65
|
+
result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx)
|
|
66
|
+
Engine::Codepoints.from_codepoints(result)
|
|
67
|
+
end
|
|
68
|
+
private_class_method :transform_text
|
|
69
|
+
|
|
70
|
+
def self.transform_html(input, locale, rules)
|
|
71
|
+
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
72
|
+
spans = Modes::Html.html_spans(input)
|
|
73
|
+
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale)
|
|
74
|
+
cp = Engine::Codepoints.to_codepoints(input)
|
|
75
|
+
Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx)
|
|
76
|
+
end
|
|
77
|
+
private_class_method :transform_html
|
|
78
|
+
|
|
79
|
+
def self.transform_markdown(input, locale, dialect, rules)
|
|
80
|
+
# Validation order is public, tested behaviour, identical across every runtime: rules (an
|
|
81
|
+
# unknown rule id), then locale (an unknown locale), then dialect/parsing.
|
|
82
|
+
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
|
|
83
|
+
require_relative "polytypo/modes/markdown"
|
|
84
|
+
Modes::Markdown.resolve_dialect(dialect)
|
|
85
|
+
spans = Modes::Markdown.markdown_spans(input)
|
|
86
|
+
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale)
|
|
87
|
+
cp = Engine::Codepoints.to_codepoints(input)
|
|
88
|
+
Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx)
|
|
89
|
+
end
|
|
90
|
+
private_class_method :transform_markdown
|
|
91
|
+
end
|
data/polytypo.gemspec
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/polytypo/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "polytypo"
|
|
7
|
+
spec.version = Polytypo::VERSION
|
|
8
|
+
spec.authors = ["Iurii Rogulia"]
|
|
9
|
+
spec.email = ["iurii@rogulia.fi"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "Locale-correct microtypography: quotes, dashes, ellipses, apostrophes, symbols and no-break spaces"
|
|
12
|
+
spec.description = "polytypo is a locale-correct microtypography engine -- quotes, dashes, " \
|
|
13
|
+
"ellipses, apostrophes, symbols and no-break spaces -- driven by a spec " \
|
|
14
|
+
"shared across every polytypo runtime (github.com/polytypo/polytypo)."
|
|
15
|
+
spec.homepage = "https://polytypo.dev/"
|
|
16
|
+
spec.license = "MIT"
|
|
17
|
+
spec.required_ruby_version = ">= 3.2"
|
|
18
|
+
|
|
19
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
|
+
spec.metadata["source_code_uri"] = "https://github.com/polytypo/polytypo-ruby"
|
|
21
|
+
spec.metadata["changelog_uri"] = "https://github.com/polytypo/polytypo-ruby/releases"
|
|
22
|
+
spec.metadata["rubygems_mfa_required"] = "true"
|
|
23
|
+
|
|
24
|
+
# Everything under lib/, plus top-level docs. Never RSpec's own spec/ directory (test-only,
|
|
25
|
+
# not part of the published gem) or CI/dev-only files.
|
|
26
|
+
spec.files = Dir.chdir(__dir__) do
|
|
27
|
+
`git ls-files -z`.split("\x0").select do |f|
|
|
28
|
+
f.start_with?("lib/") || %w[README.md LICENSE polytypo.gemspec].include?(f)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
spec.require_paths = ["lib"]
|
|
32
|
+
|
|
33
|
+
spec.add_dependency "commonmarker", "~> 2.0"
|
|
34
|
+
# Development dependencies live in the Gemfile, not here (Gemspec/DevelopmentDependencies):
|
|
35
|
+
# a gemspec's dependencies ship as metadata with every install, while dev-only tooling like
|
|
36
|
+
# rspec/rubocop belongs only to this repository's own Bundler group.
|
|
37
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: polytypo
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 1.0.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Iurii Rogulia
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: commonmarker
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '2.0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '2.0'
|
|
26
|
+
description: polytypo is a locale-correct microtypography engine -- quotes, dashes,
|
|
27
|
+
ellipses, apostrophes, symbols and no-break spaces -- driven by a spec shared across
|
|
28
|
+
every polytypo runtime (github.com/polytypo/polytypo).
|
|
29
|
+
email:
|
|
30
|
+
- iurii@rogulia.fi
|
|
31
|
+
executables: []
|
|
32
|
+
extensions: []
|
|
33
|
+
extra_rdoc_files: []
|
|
34
|
+
files:
|
|
35
|
+
- LICENSE
|
|
36
|
+
- README.md
|
|
37
|
+
- lib/polytypo.rb
|
|
38
|
+
- lib/polytypo/data/README.md
|
|
39
|
+
- lib/polytypo/data/UNICODE
|
|
40
|
+
- lib/polytypo/data/VERSION
|
|
41
|
+
- lib/polytypo/data/fixtures/de-CH.json
|
|
42
|
+
- lib/polytypo/data/fixtures/de-DE.json
|
|
43
|
+
- lib/polytypo/data/fixtures/el.json
|
|
44
|
+
- lib/polytypo/data/fixtures/en-GB.json
|
|
45
|
+
- lib/polytypo/data/fixtures/en-US.json
|
|
46
|
+
- lib/polytypo/data/fixtures/fi.json
|
|
47
|
+
- lib/polytypo/data/fixtures/fr-CA.json
|
|
48
|
+
- lib/polytypo/data/fixtures/fr.json
|
|
49
|
+
- lib/polytypo/data/fixtures/locale-resolution.json
|
|
50
|
+
- lib/polytypo/data/fixtures/ru.json
|
|
51
|
+
- lib/polytypo/data/fixtures/sv.json
|
|
52
|
+
- lib/polytypo/data/locales/de-CH.json
|
|
53
|
+
- lib/polytypo/data/locales/de-DE.json
|
|
54
|
+
- lib/polytypo/data/locales/el.json
|
|
55
|
+
- lib/polytypo/data/locales/en-GB.json
|
|
56
|
+
- lib/polytypo/data/locales/en-US.json
|
|
57
|
+
- lib/polytypo/data/locales/fi.json
|
|
58
|
+
- lib/polytypo/data/locales/fr-CA.json
|
|
59
|
+
- lib/polytypo/data/locales/fr.json
|
|
60
|
+
- lib/polytypo/data/locales/registry.json
|
|
61
|
+
- lib/polytypo/data/locales/ru.json
|
|
62
|
+
- lib/polytypo/data/locales/sv.json
|
|
63
|
+
- lib/polytypo/data/rules/dashes.md
|
|
64
|
+
- lib/polytypo/data/rules/order.json
|
|
65
|
+
- lib/polytypo/data/schema/fixtures.schema.json
|
|
66
|
+
- lib/polytypo/data/schema/locale.schema.json
|
|
67
|
+
- lib/polytypo/data/schema/registry.schema.json
|
|
68
|
+
- lib/polytypo/data/schema/resolution.schema.json
|
|
69
|
+
- lib/polytypo/engine/codepoints.rb
|
|
70
|
+
- lib/polytypo/engine/edits.rb
|
|
71
|
+
- lib/polytypo/engine/locale.rb
|
|
72
|
+
- lib/polytypo/engine/pipeline.rb
|
|
73
|
+
- lib/polytypo/engine/registry.rb
|
|
74
|
+
- lib/polytypo/engine/rules.rb
|
|
75
|
+
- lib/polytypo/engine/rules/apostrophe.rb
|
|
76
|
+
- lib/polytypo/engine/rules/dash_shared.rb
|
|
77
|
+
- lib/polytypo/engine/rules/dashes.rb
|
|
78
|
+
- lib/polytypo/engine/rules/ellipsis.rb
|
|
79
|
+
- lib/polytypo/engine/rules/hyphen.rb
|
|
80
|
+
- lib/polytypo/engine/rules/nbsp.rb
|
|
81
|
+
- lib/polytypo/engine/rules/quote_ambiguity.rb
|
|
82
|
+
- lib/polytypo/engine/rules/quotes.rb
|
|
83
|
+
- lib/polytypo/engine/rules/ranges.rb
|
|
84
|
+
- lib/polytypo/engine/rules/spaces.rb
|
|
85
|
+
- lib/polytypo/engine/rules/symbols.rb
|
|
86
|
+
- lib/polytypo/engine/sentinels.rb
|
|
87
|
+
- lib/polytypo/engine/unicode_util.rb
|
|
88
|
+
- lib/polytypo/errors.rb
|
|
89
|
+
- lib/polytypo/modes/html.rb
|
|
90
|
+
- lib/polytypo/modes/markdown.rb
|
|
91
|
+
- lib/polytypo/modes/parse_error.rb
|
|
92
|
+
- lib/polytypo/modes/runner.rb
|
|
93
|
+
- lib/polytypo/modes/spans.rb
|
|
94
|
+
- lib/polytypo/version.rb
|
|
95
|
+
- polytypo.gemspec
|
|
96
|
+
homepage: https://polytypo.dev/
|
|
97
|
+
licenses:
|
|
98
|
+
- MIT
|
|
99
|
+
metadata:
|
|
100
|
+
homepage_uri: https://polytypo.dev/
|
|
101
|
+
source_code_uri: https://github.com/polytypo/polytypo-ruby
|
|
102
|
+
changelog_uri: https://github.com/polytypo/polytypo-ruby/releases
|
|
103
|
+
rubygems_mfa_required: 'true'
|
|
104
|
+
rdoc_options: []
|
|
105
|
+
require_paths:
|
|
106
|
+
- lib
|
|
107
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
108
|
+
requirements:
|
|
109
|
+
- - ">="
|
|
110
|
+
- !ruby/object:Gem::Version
|
|
111
|
+
version: '3.2'
|
|
112
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
113
|
+
requirements:
|
|
114
|
+
- - ">="
|
|
115
|
+
- !ruby/object:Gem::Version
|
|
116
|
+
version: '0'
|
|
117
|
+
requirements: []
|
|
118
|
+
rubygems_version: 3.6.9
|
|
119
|
+
specification_version: 4
|
|
120
|
+
summary: 'Locale-correct microtypography: quotes, dashes, ellipses, apostrophes, symbols
|
|
121
|
+
and no-break spaces'
|
|
122
|
+
test_files: []
|