toon-fu 4.1.4 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +9 -9
- data/lib/toon_fu/decoder.rb +16 -10
- data/lib/toon_fu/field_list.rb +7 -2
- data/lib/toon_fu/header.rb +6 -6
- data/lib/toon_fu/lines.rb +7 -9
- data/lib/toon_fu/reader.rb +77 -108
- data/lib/toon_fu/string_literal.rb +4 -0
- data/lib/toon_fu/tokens.rb +8 -7
- data/lib/toon_fu/version.rb +1 -1
- data/lib/toon_fu/writer.rb +1 -0
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fe7b6913ab8e4bd4224ed1b08175c8ae7b2e2a5b841c990daa77f48819f4f495
|
|
4
|
+
data.tar.gz: 536f77da3c7fe3ce3a34c0d7421d5777559a8a912ab6194582be07bff441e0e1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: be1b854a3300c9e7b70cd2fa73d2d9d6ace099c88c32d1f98a3d30152a2f2c03ab42de0f23b48bbacb6eccfa042946add466eb99dec2c7e3636cb0089857e0f8
|
|
7
|
+
data.tar.gz: 40cad3826f9fd815fd636fb4ac2c6b3b4e7563fefa73a9cbbfe1444161590222f914557ecaaf9d142a680b6a741cea300579e82cf5a711a943044f7adb614a83
|
data/README.md
CHANGED
|
@@ -138,7 +138,7 @@ ToonFu.decode("tags[3]: a,b")
|
|
|
138
138
|
|
|
139
139
|
### Options
|
|
140
140
|
|
|
141
|
-
- `strict:` — `true` (default) checks
|
|
141
|
+
- `strict:` — `true` (default) checks everything the spec requires. Pass `false` for the spec's five recoveries and nothing more: a declared length is ignored, a duplicate key takes its last value, a tab indents one level, a blank line between an array's items or rows is skipped, and a scope's first line sets its depth. Everything else — a malformed header, a row of the wrong width, a stray over-indented line — raises in either mode.
|
|
142
142
|
- `indent_size:` — spaces per nesting level, default `2`.
|
|
143
143
|
|
|
144
144
|
```ruby
|
|
@@ -166,17 +166,17 @@ b08ea2d feat: ship a toon executable
|
|
|
166
166
|
|
|
167
167
|
## Compared with other Ruby TOON gems
|
|
168
168
|
|
|
169
|
-
The only one that passes every spec fixture: toon-fu passes all
|
|
169
|
+
The only one that passes every spec fixture: toon-fu passes all 160 encode and all 404 decode fixtures. The table compares encoding — the 138 encode fixtures that use default options; the other 22 set a delimiter or an indent size, which the other gems do not take, and none of them reads TOON back.
|
|
170
170
|
|
|
171
171
|
| Gem | Spec fixtures passed | Speed vs toon-fu |
|
|
172
172
|
|---|---:|---:|
|
|
173
|
-
| **toon-fu** | **
|
|
174
|
-
| sorbet-toon 0.1.0 |
|
|
175
|
-
| toon-ruby 0.1.1 |
|
|
176
|
-
| toon_my_json 0.1.0 |
|
|
177
|
-
| toon-format 0.1.2 |
|
|
173
|
+
| **toon-fu** | **138 / 138** | **1.00×** |
|
|
174
|
+
| sorbet-toon 0.1.0 | 104 / 138 | 0.55× |
|
|
175
|
+
| toon-ruby 0.1.1 | 100 / 138 | 0.55× |
|
|
176
|
+
| toon_my_json 0.1.0 | 46 / 138 | 1.49× |
|
|
177
|
+
| toon-format 0.1.2 | 40 / 138 | 1.11× |
|
|
178
178
|
|
|
179
|
-
The Ruby TOON encoders with more than 10,000 downloads, measured by [`benchmark/run.rb`](benchmark/run.rb) on Ruby 3.4.
|
|
179
|
+
The Ruby TOON encoders with more than 10,000 downloads, measured by [`benchmark/run.rb`](benchmark/run.rb) on Ruby 3.4.11 (2026-10-10). **Spec fixtures** are the spec's own encode fixtures that use default options. **Speed** is the geometric mean of encodes per second over five workloads — tables of 100 and 1000 rows, nested objects, a list of mixed objects, strings needing quotes — relative to toon-fu.
|
|
180
180
|
|
|
181
181
|
What falls through the gaps:
|
|
182
182
|
|
|
@@ -203,7 +203,7 @@ The gem version tracks the TOON specification it implements:
|
|
|
203
203
|
- `MAJOR.MINOR` is the spec version: `X.Y.Z` speaks TOON `X.Y`.
|
|
204
204
|
- `PATCH` is the gem's own: fixes and improvements within the same dialect.
|
|
205
205
|
|
|
206
|
-
This release speaks `toon-spec: 4.
|
|
206
|
+
This release speaks `toon-spec: 4.4`.
|
|
207
207
|
|
|
208
208
|
Pin the spec line with `gem "toon-fu", "~> X.Y.0"`: it takes our fixes and keeps you on the dialect you speak. The gem badge above shows the current release; the spec-drift badge turns red when a newer spec is released and toon-fu has not caught up yet.
|
|
209
209
|
|
data/lib/toon_fu/decoder.rb
CHANGED
|
@@ -3,10 +3,14 @@
|
|
|
3
3
|
module ToonFu
|
|
4
4
|
# Decodes TOON with one set of options, for callers that decode many.
|
|
5
5
|
class Decoder
|
|
6
|
-
# @param strict [Boolean] whether to enforce the spec
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
6
|
+
# @param strict [Boolean] whether to enforce every check of the spec;
|
|
7
|
+
# with +false+ exactly the spec's five recoveries apply: a declared
|
|
8
|
+
# length is ignored; a duplicate key or a field name repeated in one
|
|
9
|
+
# field list resolves last-write-wins, keeping its first position; a
|
|
10
|
+
# tab in indentation counts as one level, leading spaces need not be a
|
|
11
|
+
# multiple of indent_size and a line of spaces and tabs is blank; a
|
|
12
|
+
# blank line inside a header span is ignored; and a scope's first line
|
|
13
|
+
# sets the scope's depth
|
|
10
14
|
# @param indent_size [Integer] spaces per nesting level
|
|
11
15
|
# @raise [ArgumentError] when strict is not a boolean or indent_size is
|
|
12
16
|
# not a positive Integer
|
|
@@ -40,12 +44,14 @@ module ToonFu
|
|
|
40
44
|
# @raise [Error] for text that is not a String or not valid UTF-8; for a
|
|
41
45
|
# number outside the Float range; for an invalid escape, an
|
|
42
46
|
# unterminated quoted string, or content after a closing quote; for a
|
|
43
|
-
# key without a colon; for a
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
47
|
+
# key without a colon; for a malformed or misplaced header; for a row
|
|
48
|
+
# width that does not match; for a line that belongs to no scope, an
|
|
49
|
+
# over-indented or indented first line included; for content after a
|
|
50
|
+
# completed root form; for nesting beyond the depth limit; and, when
|
|
51
|
+
# strict, for a declared length that does not match, a duplicate key, a
|
|
52
|
+
# field name repeated in one field list, a blank line inside a header
|
|
53
|
+
# span, a tab in indentation, indentation that is not a multiple of
|
|
54
|
+
# indent_size, and a depth jump
|
|
49
55
|
def decode(text)
|
|
50
56
|
Reader.new(@strict, @indent_size).read(utf8(text))
|
|
51
57
|
end
|
data/lib/toon_fu/field_list.rb
CHANGED
|
@@ -37,16 +37,21 @@ module ToonFu
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def split(entry)
|
|
40
|
+
entry = Tokens.trim(entry)
|
|
41
|
+
return [nil, nil] if entry.empty?
|
|
42
|
+
|
|
40
43
|
brace = Tokens.brace_index(entry)
|
|
41
|
-
return [
|
|
44
|
+
return [Token.key(entry), nil] if brace.nil?
|
|
42
45
|
|
|
43
46
|
name = entry[0...brace]
|
|
47
|
+
return [nil, nil] if name.empty? || name.match?(Tokens::TRAILING_WHITESPACE)
|
|
48
|
+
|
|
44
49
|
group = entry[brace..]
|
|
45
50
|
closing = Tokens.closing_brace(group)
|
|
46
51
|
return [nil, nil] unless closing == group.length - 1
|
|
47
52
|
|
|
48
53
|
nested = self.class.parse(group[1...closing], @delimiter)
|
|
49
|
-
nested.nil? ? [nil, nil] : [
|
|
54
|
+
nested.nil? ? [nil, nil] : [Token.key(name), nested]
|
|
50
55
|
end
|
|
51
56
|
end
|
|
52
57
|
end
|
data/lib/toon_fu/header.rb
CHANGED
|
@@ -9,7 +9,7 @@ module ToonFu
|
|
|
9
9
|
|
|
10
10
|
def self.parse(content)
|
|
11
11
|
bracket = Tokens.bracket_index(content)
|
|
12
|
-
return nil if bracket.nil?
|
|
12
|
+
return nil if bracket.nil? || Tokens.colon_index(content).nil?
|
|
13
13
|
|
|
14
14
|
new(content, bracket).parse
|
|
15
15
|
end
|
|
@@ -20,9 +20,9 @@ module ToonFu
|
|
|
20
20
|
@malformed = false
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
# A line
|
|
24
|
-
# one that §6
|
|
25
|
-
#
|
|
23
|
+
# A line whose first unquoted "[" precedes its first unquoted colon is an
|
|
24
|
+
# array-header line; one that fails the §6 grammar is malformed, an
|
|
25
|
+
# error in either mode (§5.2, §14.2).
|
|
26
26
|
def malformed?
|
|
27
27
|
@malformed
|
|
28
28
|
end
|
|
@@ -30,7 +30,7 @@ module ToonFu
|
|
|
30
30
|
def parse
|
|
31
31
|
@key = @bracket.zero? ? nil : @content[0...@bracket]
|
|
32
32
|
return nil unless @key.nil? || Tokens.whole_token?(@key)
|
|
33
|
-
return broken if @key&.match?(
|
|
33
|
+
return broken if @key&.match?(Tokens::TRAILING_WHITESPACE)
|
|
34
34
|
return broken unless (closing = @content.index("]", @bracket))
|
|
35
35
|
|
|
36
36
|
return broken unless segment(@content[(@bracket + 1)...closing])
|
|
@@ -38,7 +38,7 @@ module ToonFu
|
|
|
38
38
|
# §6: a keyed header's field list is required, and a fields-bearing
|
|
39
39
|
# header carries no inline content.
|
|
40
40
|
return broken if @keyed && !fields?
|
|
41
|
-
return broken if fields? &&
|
|
41
|
+
return broken if fields? && !Tokens.trim(@inline).empty?
|
|
42
42
|
|
|
43
43
|
self
|
|
44
44
|
end
|
data/lib/toon_fu/lines.rb
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module ToonFu
|
|
4
4
|
class Line
|
|
5
|
-
ITEM = /\A-(?:
|
|
5
|
+
ITEM = /\A-(?: +|\z)/
|
|
6
6
|
|
|
7
7
|
attr_reader :content, :depth, :number, :text
|
|
8
8
|
|
|
@@ -29,8 +29,8 @@ module ToonFu
|
|
|
29
29
|
BOM = "\u{feff}"
|
|
30
30
|
COMMENT = /\A *#/
|
|
31
31
|
TRAILING_SPACES = / +\z/
|
|
32
|
-
BLANK = /\A\s*\z/
|
|
33
32
|
INDENTATION = /\A[ \t]*/
|
|
33
|
+
BLANK_WITH_TABS = /\A[ \t]*\z/
|
|
34
34
|
|
|
35
35
|
def initialize(text, strict, indent_size)
|
|
36
36
|
@strict = strict
|
|
@@ -57,10 +57,6 @@ module ToonFu
|
|
|
57
57
|
line
|
|
58
58
|
end
|
|
59
59
|
|
|
60
|
-
def number
|
|
61
|
-
peek ? peek.number : @lines.length
|
|
62
|
-
end
|
|
63
|
-
|
|
64
60
|
private
|
|
65
61
|
|
|
66
62
|
def significant(text)
|
|
@@ -69,7 +65,7 @@ module ToonFu
|
|
|
69
65
|
line = raw.delete_suffix("\r").sub(TRAILING_SPACES, "")
|
|
70
66
|
next if COMMENT.match?(line)
|
|
71
67
|
|
|
72
|
-
if
|
|
68
|
+
if blank?(line)
|
|
73
69
|
blank = true
|
|
74
70
|
next
|
|
75
71
|
end
|
|
@@ -80,12 +76,14 @@ module ToonFu
|
|
|
80
76
|
end
|
|
81
77
|
end
|
|
82
78
|
|
|
79
|
+
def blank?(line)
|
|
80
|
+
@strict ? line.empty? : BLANK_WITH_TABS.match?(line)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
83
|
def build(line, number, blank)
|
|
84
84
|
Line.new(line.sub(INDENTATION, ""), depth(line, number), number, line, blank)
|
|
85
85
|
end
|
|
86
86
|
|
|
87
|
-
# Non-strict mode accepts a tab as indentation (§12), where each tab
|
|
88
|
-
# counts as one level; strict mode rejects it.
|
|
89
87
|
def depth(line, number)
|
|
90
88
|
indent = line[INDENTATION]
|
|
91
89
|
raise Error, "cannot decode a line indented with a tab on line #{number}: #{line}" if @strict && indent.include?("\t")
|
data/lib/toon_fu/reader.rb
CHANGED
|
@@ -8,6 +8,7 @@ module ToonFu
|
|
|
8
8
|
@strict = strict
|
|
9
9
|
@indent_size = indent_size
|
|
10
10
|
@depth = 0
|
|
11
|
+
@in_span = false
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
def read(text)
|
|
@@ -21,18 +22,19 @@ module ToonFu
|
|
|
21
22
|
|
|
22
23
|
def root
|
|
23
24
|
line = @lines.peek
|
|
25
|
+
return object(-1, 0) unless line.depth.zero?
|
|
24
26
|
return empty_array_root if line.content == "[]"
|
|
25
27
|
|
|
26
|
-
header =
|
|
28
|
+
header = parse_header(line)
|
|
27
29
|
return keyless_root(header) if header && header.key.nil?
|
|
28
|
-
return scalar_root(line) if @lines.one? &&
|
|
30
|
+
return scalar_root(line) if @lines.one? && scalar?(line)
|
|
29
31
|
|
|
30
|
-
object(0)
|
|
32
|
+
object(-1, 0)
|
|
31
33
|
end
|
|
32
34
|
|
|
33
35
|
def keyless_root(header)
|
|
34
36
|
@lines.next
|
|
35
|
-
value = header.fields? ? table(header,
|
|
37
|
+
value = header.fields? ? table(header, 0) : array(header, 0)
|
|
36
38
|
trailing
|
|
37
39
|
value
|
|
38
40
|
end
|
|
@@ -54,24 +56,25 @@ module ToonFu
|
|
|
54
56
|
raise Error, "cannot decode content after the root form: #{@lines.peek.text}"
|
|
55
57
|
end
|
|
56
58
|
|
|
57
|
-
def object(depth, result = {})
|
|
58
|
-
while (line = @lines.peek) && line.depth
|
|
59
|
-
if line.depth
|
|
59
|
+
def object(parent, depth, result = {})
|
|
60
|
+
while (line = @lines.peek) && line.depth > parent
|
|
61
|
+
if line.depth == depth
|
|
62
|
+
take
|
|
63
|
+
key, value = field(line, depth)
|
|
64
|
+
store(result, key, value, line)
|
|
65
|
+
else
|
|
60
66
|
orphan(line)
|
|
61
|
-
next
|
|
62
67
|
end
|
|
63
|
-
|
|
64
|
-
@lines.next
|
|
65
|
-
key, value = field(line, depth)
|
|
66
|
-
store(result, key, value, line)
|
|
67
68
|
end
|
|
68
69
|
result
|
|
69
70
|
end
|
|
70
71
|
|
|
71
72
|
def orphan(line)
|
|
72
|
-
raise Error, "cannot decode a line that belongs to no scope on line #{line.number}: #{line.text}"
|
|
73
|
+
raise Error, "cannot decode a line that belongs to no scope on line #{line.number}: #{line.text}"
|
|
74
|
+
end
|
|
73
75
|
|
|
74
|
-
|
|
76
|
+
def scalar?(line)
|
|
77
|
+
Tokens.colon_index(line.content).nil?
|
|
75
78
|
end
|
|
76
79
|
|
|
77
80
|
def store(result, key, value, line)
|
|
@@ -81,12 +84,12 @@ module ToonFu
|
|
|
81
84
|
end
|
|
82
85
|
|
|
83
86
|
def field(line, depth)
|
|
84
|
-
header =
|
|
87
|
+
header = parse_header(line)
|
|
85
88
|
return [Tokens.key(header.key), header_value(header, depth)] if header&.key
|
|
86
|
-
|
|
89
|
+
misplaced(line, "a keyless array header in object field position") if header
|
|
87
90
|
|
|
88
91
|
colon = Tokens.colon_index(line.content)
|
|
89
|
-
raise Error, "cannot decode a line without a colon
|
|
92
|
+
raise Error, "cannot decode a line without a colon on line #{line.number}: #{line.text}" if colon.nil?
|
|
90
93
|
|
|
91
94
|
key = Tokens.key(line.content[0...colon])
|
|
92
95
|
[key, field_value(Tokens.trim(line.content[(colon + 1)..]), depth)]
|
|
@@ -99,30 +102,23 @@ module ToonFu
|
|
|
99
102
|
Token.decode(text)
|
|
100
103
|
end
|
|
101
104
|
|
|
102
|
-
def nested(
|
|
103
|
-
|
|
104
|
-
return {} if
|
|
105
|
+
def nested(parent)
|
|
106
|
+
depth = content_depth(parent)
|
|
107
|
+
return {} if depth.nil?
|
|
105
108
|
|
|
106
|
-
|
|
107
|
-
descend { object(depth + 1) }
|
|
109
|
+
descend { object(parent, depth) }
|
|
108
110
|
end
|
|
109
111
|
|
|
110
112
|
def header_value(header, depth)
|
|
111
|
-
|
|
112
|
-
return [] if header.length.zero? && !items?(depth + 1)
|
|
113
|
-
|
|
114
|
-
array(header, depth + 1)
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def items?(depth)
|
|
118
|
-
line = @lines.peek
|
|
119
|
-
!line.nil? && line.depth == depth
|
|
113
|
+
header.fields? ? table(header, depth) : array(header, depth)
|
|
120
114
|
end
|
|
121
115
|
|
|
122
|
-
def array(header,
|
|
116
|
+
def array(header, parent)
|
|
123
117
|
return inline(header) unless header.inline.empty?
|
|
124
118
|
|
|
125
|
-
descend {
|
|
119
|
+
items = descend { gather(parent, stop: ->(line) { !line.item? }) { |line| item(line) } }
|
|
120
|
+
count(items.length, header, "item")
|
|
121
|
+
items
|
|
126
122
|
end
|
|
127
123
|
|
|
128
124
|
def inline(header)
|
|
@@ -131,67 +127,47 @@ module ToonFu
|
|
|
131
127
|
cells
|
|
132
128
|
end
|
|
133
129
|
|
|
134
|
-
def
|
|
135
|
-
items = []
|
|
136
|
-
while (line = @lines.peek) && line.depth == depth && line.item?
|
|
137
|
-
span(line) unless items.empty?
|
|
138
|
-
@lines.next
|
|
139
|
-
items << item(line, depth)
|
|
140
|
-
end
|
|
141
|
-
scalar_line(@lines.peek, depth)
|
|
142
|
-
count(items.length, header, "item")
|
|
143
|
-
items
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
# A scalar line is valid only as a root primitive; inside a scope it is a
|
|
147
|
-
# structural error in strict and non-strict mode alike (§5.2, §14.2).
|
|
148
|
-
def scalar_line(line, depth)
|
|
149
|
-
return if line.nil? || line.depth != depth || Tokens.colon_index(line.content)
|
|
150
|
-
|
|
151
|
-
header = Header.parse(line.content)
|
|
152
|
-
return if header && !header.malformed?
|
|
153
|
-
|
|
154
|
-
raise Error, "cannot decode a bare token line inside a scope on line #{line.number}: #{line.text}"
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
def item(line, depth)
|
|
130
|
+
def item(line)
|
|
158
131
|
rest = line.item_content
|
|
159
132
|
return {} if rest.empty?
|
|
160
133
|
return [] if rest == "[]"
|
|
161
134
|
|
|
162
|
-
header =
|
|
163
|
-
return item_header(header, line
|
|
164
|
-
return item_object(rest,
|
|
135
|
+
header = parse_header(line, rest)
|
|
136
|
+
return item_header(header, line) if header
|
|
137
|
+
return item_object(rest, line) if Tokens.colon_index(rest)
|
|
165
138
|
|
|
166
139
|
Token.decode(Tokens.trim(rest))
|
|
167
140
|
end
|
|
168
141
|
|
|
169
|
-
def item_header(header, line
|
|
170
|
-
return
|
|
171
|
-
raise Error, "cannot decode a keyless fields-bearing header as a list item on line #{line.number}: #{line.text}" if header.fields?
|
|
142
|
+
def item_header(header, line)
|
|
143
|
+
return array(header, line.depth) unless header.key || header.fields?
|
|
172
144
|
|
|
173
|
-
|
|
145
|
+
misplaced(line, "a keyless fields-bearing header as a list item") unless header.key
|
|
146
|
+
item_object(line.item_content, line)
|
|
174
147
|
end
|
|
175
148
|
|
|
176
|
-
def
|
|
177
|
-
|
|
178
|
-
|
|
149
|
+
def misplaced(line, what)
|
|
150
|
+
raise Error, "cannot decode #{what} on line #{line.number}: #{line.text}"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def item_object(rest, line)
|
|
154
|
+
depth = line.depth + 1
|
|
155
|
+
synthetic = Line.new(rest, depth, line.number)
|
|
156
|
+
key, value = descend { field(synthetic, depth) }
|
|
179
157
|
result = {}
|
|
180
158
|
store(result, key, value, synthetic)
|
|
181
|
-
|
|
182
|
-
span(following) if following && following.depth == depth + 1
|
|
183
|
-
descend { object(depth + 1, result) }
|
|
159
|
+
descend { object(line.depth, depth, result) }
|
|
184
160
|
end
|
|
185
161
|
|
|
186
|
-
def table(header,
|
|
162
|
+
def table(header, parent)
|
|
187
163
|
raise Error, "cannot decode a field name repeated in one field list: #{header.duplicate.inspect}" if @strict && header.duplicate
|
|
188
164
|
|
|
189
|
-
header.keyed? ? entries(header,
|
|
165
|
+
header.keyed? ? entries(header, parent) : rows(header, parent)
|
|
190
166
|
end
|
|
191
167
|
|
|
192
|
-
def rows(header,
|
|
168
|
+
def rows(header, parent)
|
|
193
169
|
collected = descend do
|
|
194
|
-
gather(
|
|
170
|
+
gather(parent, stop: ->(line) { key_value_line?(line, header.delimiter) }) { |line| row(header, line) }
|
|
195
171
|
end
|
|
196
172
|
count(collected.length, header, "row")
|
|
197
173
|
collected
|
|
@@ -207,62 +183,53 @@ module ToonFu
|
|
|
207
183
|
cell.nil? || colon < cell
|
|
208
184
|
end
|
|
209
185
|
|
|
210
|
-
def entries(header,
|
|
186
|
+
def entries(header, parent)
|
|
211
187
|
result = {}
|
|
212
188
|
collected = descend do
|
|
213
|
-
gather(
|
|
189
|
+
gather(parent) do |line|
|
|
214
190
|
colon = Tokens.colon_index(line.content)
|
|
215
|
-
if colon.nil?
|
|
216
|
-
raise Error, "cannot decode an entry row without a colon on line #{line.number}: #{line.text}" if @strict
|
|
191
|
+
raise Error, "cannot decode an entry row without a colon on line #{line.number}: #{line.text}" if colon.nil?
|
|
217
192
|
|
|
218
|
-
next nil
|
|
219
|
-
end
|
|
220
193
|
[Tokens.key(line.content[0...colon]), row(header, line, line.content[(colon + 1)..]), line]
|
|
221
194
|
end
|
|
222
195
|
end
|
|
223
|
-
collected.compact!
|
|
224
196
|
count(collected.length, header, "entry row")
|
|
225
197
|
collected.each { |key, value, line| store(result, key, value, line) }
|
|
226
198
|
result
|
|
227
199
|
end
|
|
228
200
|
|
|
229
|
-
def gather(
|
|
201
|
+
def gather(parent, stop: nil)
|
|
202
|
+
outer = @in_span
|
|
230
203
|
collected = []
|
|
231
|
-
|
|
204
|
+
depth = content_depth(parent)
|
|
205
|
+
while depth && (line = @lines.peek) && line.depth > parent
|
|
206
|
+
orphan(line) if line.depth != depth
|
|
232
207
|
break if stop&.call(line)
|
|
233
208
|
|
|
234
|
-
|
|
235
|
-
@
|
|
209
|
+
take
|
|
210
|
+
@in_span = true
|
|
236
211
|
collected << yield(line)
|
|
237
212
|
end
|
|
238
213
|
collected
|
|
214
|
+
ensure
|
|
215
|
+
@in_span = outer
|
|
239
216
|
end
|
|
240
217
|
|
|
241
|
-
def
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
raise Error, "cannot decode a blank line inside a header span, before line #{line.number}: #{line.text}"
|
|
218
|
+
def take
|
|
219
|
+
line = @lines.next
|
|
220
|
+
raise Error, "cannot decode a blank line inside a header span, before line #{line.number}: #{line.text}" if @strict && @in_span && line.after_blank
|
|
245
221
|
end
|
|
246
222
|
|
|
247
223
|
def row(header, line, text = line.content)
|
|
248
224
|
cells = Tokens.split(text, header.delimiter)
|
|
249
225
|
cells = [] if cells.length == 1 && Tokens.trim(cells.first).empty?
|
|
250
226
|
width(cells.length, header, line)
|
|
251
|
-
shape(header.fields, cells
|
|
227
|
+
shape(header.fields, cells)
|
|
252
228
|
end
|
|
253
229
|
|
|
254
230
|
def shape(fields, cells)
|
|
255
231
|
fields.each_with_object({}) do |(name, nested), result|
|
|
256
|
-
|
|
257
|
-
result[name] = shape(nested, cells)
|
|
258
|
-
else
|
|
259
|
-
cell = begin
|
|
260
|
-
cells.next
|
|
261
|
-
rescue StopIteration
|
|
262
|
-
next
|
|
263
|
-
end
|
|
264
|
-
result[name] = Token.decode(Tokens.trim(cell))
|
|
265
|
-
end
|
|
232
|
+
result[name] = nested ? shape(nested, cells) : Token.decode(Tokens.trim(cells.shift))
|
|
266
233
|
end
|
|
267
234
|
end
|
|
268
235
|
|
|
@@ -273,22 +240,24 @@ module ToonFu
|
|
|
273
240
|
end
|
|
274
241
|
|
|
275
242
|
def width(actual, header, line)
|
|
276
|
-
return
|
|
243
|
+
return if actual == header.leaves.length
|
|
277
244
|
|
|
278
245
|
raise Error, "cannot decode #{actual} cells on line #{line.number} where the header declares #{header.leaves.length} field#{"s" unless header.leaves.length == 1}"
|
|
279
246
|
end
|
|
280
247
|
|
|
281
|
-
def
|
|
282
|
-
|
|
283
|
-
raise Error, "cannot decode a malformed array header on line #{line.number}: #{line.text}" if
|
|
248
|
+
def parse_header(line, content = line.content)
|
|
249
|
+
header = Header.parse(content)
|
|
250
|
+
raise Error, "cannot decode a malformed array header on line #{line.number}: #{line.text}" if header&.malformed?
|
|
284
251
|
|
|
285
|
-
|
|
252
|
+
header
|
|
286
253
|
end
|
|
287
254
|
|
|
288
|
-
def
|
|
289
|
-
|
|
255
|
+
def content_depth(parent)
|
|
256
|
+
line = @lines.peek
|
|
257
|
+
return if line.nil? || line.depth <= parent
|
|
258
|
+
raise Error, "cannot decode a depth jump on line #{line.number}: #{line.text}" if @strict && line.depth > parent + 1
|
|
290
259
|
|
|
291
|
-
|
|
260
|
+
line.depth
|
|
292
261
|
end
|
|
293
262
|
|
|
294
263
|
def descend
|
data/lib/toon_fu/tokens.rb
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module ToonFu
|
|
4
4
|
module Tokens
|
|
5
|
-
|
|
5
|
+
QUOTE_SPAN = /"(?:[^"\\]|\\.?)*"?/
|
|
6
|
+
TRAILING_WHITESPACE = /[ \t]\z/
|
|
6
7
|
UNTIL_COLON = /[^":]+/
|
|
7
8
|
UNTIL_BRACKET = /[^":\[]+/
|
|
8
9
|
UNTIL_BRACE = /[^"{]+/
|
|
@@ -17,7 +18,7 @@ module ToonFu
|
|
|
17
18
|
until scanner.eos?
|
|
18
19
|
next if scanner.skip(pattern)
|
|
19
20
|
return scanner.charpos if stops.include?(scanner.peek(1))
|
|
20
|
-
|
|
21
|
+
scanner.skip(QUOTE_SPAN)
|
|
21
22
|
end
|
|
22
23
|
nil
|
|
23
24
|
end
|
|
@@ -40,7 +41,7 @@ module ToonFu
|
|
|
40
41
|
depth = 0
|
|
41
42
|
scanner = StringScanner.new(text)
|
|
42
43
|
until scanner.eos?
|
|
43
|
-
if scanner.skip(
|
|
44
|
+
if scanner.skip(QUOTE_SPAN)
|
|
44
45
|
next
|
|
45
46
|
elsif scanner.skip(/\{/)
|
|
46
47
|
depth += 1
|
|
@@ -48,7 +49,7 @@ module ToonFu
|
|
|
48
49
|
depth -= 1
|
|
49
50
|
return scanner.charpos - 1 if depth.zero?
|
|
50
51
|
else
|
|
51
|
-
scanner.skip(/[^"{}]+/)
|
|
52
|
+
scanner.skip(/[^"{}]+/)
|
|
52
53
|
end
|
|
53
54
|
end
|
|
54
55
|
nil
|
|
@@ -60,7 +61,7 @@ module ToonFu
|
|
|
60
61
|
depth = 0
|
|
61
62
|
scanner = StringScanner.new(text)
|
|
62
63
|
until scanner.eos?
|
|
63
|
-
if (quoted = scanner.scan(
|
|
64
|
+
if (quoted = scanner.scan(QUOTE_SPAN))
|
|
64
65
|
current << quoted
|
|
65
66
|
elsif depth.zero? && scanner.skip(delimiter)
|
|
66
67
|
entries << current
|
|
@@ -81,13 +82,13 @@ module ToonFu
|
|
|
81
82
|
current = +""
|
|
82
83
|
scanner = StringScanner.new(text)
|
|
83
84
|
until scanner.eos?
|
|
84
|
-
if (quoted = scanner.scan(
|
|
85
|
+
if (quoted = scanner.scan(QUOTE_SPAN))
|
|
85
86
|
current << quoted
|
|
86
87
|
elsif scanner.skip(delimiter)
|
|
87
88
|
cells << current
|
|
88
89
|
current = +""
|
|
89
90
|
else
|
|
90
|
-
current <<
|
|
91
|
+
current << scanner.scan(UNTIL_CELL_END.fetch(delimiter))
|
|
91
92
|
end
|
|
92
93
|
end
|
|
93
94
|
cells << current
|
data/lib/toon_fu/version.rb
CHANGED
data/lib/toon_fu/writer.rb
CHANGED