c0data 0.3.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +122 -24
- data/c0-c/c0.h +2309 -1509
- data/c0-c/c0_diff.c +551 -0
- data/c0-c/c0_diff.h +176 -0
- data/c0-c/c0_strbuf.h +49 -0
- data/ext/c0/c0_diff_impl.c +5 -0
- data/ext/c0/c0_ext.c +354 -69
- data/ext/c0/extconf.rb +2 -1
- data/lib/c0/body.rb +158 -38
- data/lib/c0/builder.rb +18 -13
- data/lib/c0/diff.rb +193 -0
- data/lib/c0/document.rb +49 -25
- data/lib/c0/stream.rb +71 -13
- data/lib/c0.rb +18 -11
- metadata +6 -1
data/lib/c0/document.rb
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module C0
|
|
4
|
-
# The heading of a document: the keyed units after SYN
|
|
5
|
-
# duplicate key. Enumerates [key, value] pairs.
|
|
4
|
+
# The heading of a document: the keyed units after SYN (dead units
|
|
5
|
+
# skipped; the heading's values take no sub-units). Last wins for a duplicate key. Enumerates [key, value] pairs.
|
|
6
6
|
class Heading
|
|
7
7
|
include Enumerable
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# The keys the format defines (DESIGN.md "Heading"): `shape` is `diff` or
|
|
10
|
+
# `stream`; `schema` a name, or a level holding one keyed record of
|
|
11
|
+
# `name`, `url`, `hash`; `version`; `blob` the blob table.
|
|
12
|
+
KNOWN_KEYS = %w[shape schema version blob].freeze
|
|
10
13
|
|
|
11
14
|
def initialize(buf, start, stop)
|
|
12
15
|
@buf = buf
|
|
@@ -15,7 +18,7 @@ module C0
|
|
|
15
18
|
end
|
|
16
19
|
|
|
17
20
|
# [key, value] in written order: the logical value, or for a nested level
|
|
18
|
-
# (the blob table) its raw bytes.
|
|
21
|
+
# (the blob table, a schema record) its raw bytes.
|
|
19
22
|
def pairs
|
|
20
23
|
@pairs ||= raw_pairs.map do |k, s, e|
|
|
21
24
|
[k, Ext.unit_type(@buf, s, e) == UnitType::LEVEL ? @buf.byteslice(s, e - s) : Ext.value(@buf, s, e)]
|
|
@@ -47,13 +50,19 @@ module C0
|
|
|
47
50
|
v && v.match?(/\A\d+\z/) ? v.to_i : 1
|
|
48
51
|
end
|
|
49
52
|
|
|
53
|
+
# The schema the body claims: a plain name (bytes), or for the keyed form
|
|
54
|
+
# the one keyed Record of `name`, `url`, `hash`; nil when absent. The
|
|
55
|
+
# format checks the form; a schema layer checks the match. The same
|
|
56
|
+
# shape in every port.
|
|
57
|
+
def schema = level("schema")&.record(0) || self["schema"]
|
|
58
|
+
|
|
50
59
|
# The value under a key as a nested level, or nil when absent or not one.
|
|
51
60
|
def level(key)
|
|
52
61
|
range = raw_range(key)
|
|
53
62
|
return nil unless range && Ext.unit_type(@buf, *range) == UnitType::LEVEL
|
|
54
63
|
|
|
55
64
|
ls, le = Ext.unit_level(@buf, *range)
|
|
56
|
-
Body.
|
|
65
|
+
Body.view(@buf, ls, le)
|
|
57
66
|
end
|
|
58
67
|
|
|
59
68
|
# The blob table as a Body, when declared as a level.
|
|
@@ -77,15 +86,25 @@ module C0
|
|
|
77
86
|
end
|
|
78
87
|
|
|
79
88
|
# One C0DATA document: an optional SYN heading, a body, an optional EOT.
|
|
80
|
-
# `Document.new(buf)`
|
|
81
|
-
#
|
|
82
|
-
#
|
|
89
|
+
# `Document.new(buf)` parses the first document in the buffer, applying the
|
|
90
|
+
# whole grammar (a C0::SyntaxError on malformed input). Use `Container` for
|
|
91
|
+
# a buffer holding several documents or an EM tail. Enumerates the groups
|
|
92
|
+
# of the body's first file.
|
|
83
93
|
class Document
|
|
84
94
|
include Enumerable
|
|
85
95
|
|
|
86
|
-
|
|
96
|
+
# The shapes the format defines; any other is a loud warning.
|
|
97
|
+
SHAPES = %w[diff stream].freeze
|
|
98
|
+
|
|
99
|
+
# Warning bits, as the C core reports them (DESIGN.md "Warnings").
|
|
100
|
+
WARN_NO_SYN = 0x01
|
|
101
|
+
WARN_NO_EOT = 0x02
|
|
102
|
+
WARN_UNKNOWN_KEY = 0x04
|
|
103
|
+
WARN_DUPLICATE_KEY = 0x08
|
|
104
|
+
WARN_UNKNOWN_SHAPE = 0x10
|
|
105
|
+
WARN_UNKNOWN_VERSION = 0x20
|
|
87
106
|
|
|
88
|
-
attr_reader :buf, :heading, :body, :start, :stop
|
|
107
|
+
attr_reader :buf, :heading, :body, :start, :stop, :warning_bits, :dead_elements
|
|
89
108
|
|
|
90
109
|
def initialize(buf, start = 0)
|
|
91
110
|
@buf = C0.to_bin(buf)
|
|
@@ -108,27 +127,29 @@ module C0
|
|
|
108
127
|
|
|
109
128
|
alias eot? has_eot?
|
|
110
129
|
|
|
111
|
-
# Loud warnings (DESIGN.md "Warnings") as strings; empty for a careful
|
|
130
|
+
# Loud warnings (DESIGN.md "Warnings") as strings; empty for a careful
|
|
131
|
+
# writer's file. Warnings never fail a parse.
|
|
112
132
|
def warnings
|
|
113
133
|
@warnings ||= begin
|
|
114
134
|
w = []
|
|
115
|
-
w << "no SYN at the start of the document"
|
|
116
|
-
w << "no EOT at the end of the document" if
|
|
117
|
-
|
|
118
|
-
w << "unknown heading key '#{k}'" unless Heading::KNOWN_KEYS.include?(k)
|
|
119
|
-
end
|
|
120
|
-
seen = {}
|
|
121
|
-
heading.keys.each { |k| w << "duplicate heading key '#{k}'" if seen.key?(k) && !w.include?("duplicate heading key '#{k}'"); seen[k] = true }
|
|
122
|
-
if (sh = heading.shape) && !SHAPES.include?(sh)
|
|
123
|
-
w << "unknown shape '#{sh}'"
|
|
135
|
+
w << "no SYN at the start of the document" if warn?(WARN_NO_SYN)
|
|
136
|
+
w << "no EOT at the end of the document" if warn?(WARN_NO_EOT)
|
|
137
|
+
if warn?(WARN_UNKNOWN_KEY)
|
|
138
|
+
heading.keys.uniq.each { |k| w << "unknown heading key '#{k}'" unless Heading::KNOWN_KEYS.include?(k) }
|
|
124
139
|
end
|
|
125
|
-
if (
|
|
126
|
-
w << "
|
|
140
|
+
if warn?(WARN_DUPLICATE_KEY)
|
|
141
|
+
heading.keys.tally.each { |k, n| w << "duplicate heading key '#{k}'" if n > 1 }
|
|
127
142
|
end
|
|
143
|
+
w << "unknown shape '#{heading.shape}'" if warn?(WARN_UNKNOWN_SHAPE)
|
|
144
|
+
w << "unknown version '#{heading['version']}'" if warn?(WARN_UNKNOWN_VERSION)
|
|
128
145
|
w
|
|
129
146
|
end
|
|
130
147
|
end
|
|
131
148
|
|
|
149
|
+
# Whether the body held any dead element (a dead record, header row,
|
|
150
|
+
# group or file); they are not in the model.
|
|
151
|
+
def dead_elements? = @dead_elements.positive?
|
|
152
|
+
|
|
132
153
|
# ---- convenience accessors over the body --------------------------------
|
|
133
154
|
|
|
134
155
|
def name = @body.name
|
|
@@ -158,15 +179,18 @@ module C0
|
|
|
158
179
|
|
|
159
180
|
private
|
|
160
181
|
|
|
182
|
+
def warn?(bit) = (@warning_bits & bit) != 0
|
|
183
|
+
|
|
161
184
|
def load(desc)
|
|
162
|
-
@start, @stop, @has_syn, @has_eot, hs, he, bs, be,
|
|
185
|
+
@start, @stop, @has_syn, @has_eot, hs, he, bs, be, @warning_bits, @dead_elements = desc
|
|
163
186
|
@heading = Heading.new(@buf, hs, he)
|
|
164
|
-
@body = Body.
|
|
187
|
+
@body = Body.view(@buf, bs, be)
|
|
165
188
|
end
|
|
166
189
|
end
|
|
167
190
|
|
|
168
191
|
# A file or message: one or more documents and an optional raw tail after
|
|
169
|
-
# EM.
|
|
192
|
+
# EM. Parsing applies the whole grammar to every document. Enumerates the
|
|
193
|
+
# documents.
|
|
170
194
|
class Container
|
|
171
195
|
include Enumerable
|
|
172
196
|
|
data/lib/c0/stream.rb
CHANGED
|
@@ -7,24 +7,46 @@ module C0
|
|
|
7
7
|
# r.torn? # => data trails the last commit
|
|
8
8
|
# r.each_record { |rec| ... }
|
|
9
9
|
module Stream
|
|
10
|
+
# A committed block that is not well-formed (DESIGN.md "Damaged Blocks").
|
|
11
|
+
# `resume_offset` is where reading can go on: after the block's ETB and
|
|
12
|
+
# payload.
|
|
13
|
+
class DamagedBlockError < C0::Error
|
|
14
|
+
attr_reader :index, :resume_offset
|
|
15
|
+
|
|
16
|
+
def initialize(index, resume_offset)
|
|
17
|
+
@index = index
|
|
18
|
+
@resume_offset = resume_offset
|
|
19
|
+
super("block #{index} is damaged; reading can resume at byte #{resume_offset}")
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
10
23
|
# Scans a buffer for commits. An unescaped ETB is always a commit, even
|
|
11
|
-
# inside open text or a level
|
|
12
|
-
#
|
|
13
|
-
#
|
|
24
|
+
# inside open text or a level. A committed block that is not well-formed,
|
|
25
|
+
# cut by its ETB or holding any other error, is **damaged**: it delivers
|
|
26
|
+
# nothing and adds nothing to the parse state. By default the reader
|
|
27
|
+
# delivers the records before the first damaged block and then raises
|
|
28
|
+
# DamagedBlockError; with `continue_past_damage: true` it skips the block
|
|
29
|
+
# and reads on. The tail after the last commit is torn unless it holds
|
|
30
|
+
# only layout, notes, EOT or EM. Enumerates the committed blocks' bytes.
|
|
14
31
|
class Reader
|
|
15
32
|
include Enumerable
|
|
16
33
|
|
|
17
|
-
attr_reader :buf, :committed_end
|
|
34
|
+
attr_reader :buf, :committed_end, :first_damaged, :resume_offset
|
|
18
35
|
|
|
19
|
-
def initialize(buf)
|
|
36
|
+
def initialize(buf, continue_past_damage: false)
|
|
20
37
|
@buf = C0.to_bin(buf)
|
|
21
|
-
@
|
|
38
|
+
@continue_past_damage = continue_past_damage
|
|
39
|
+
@committed_end, @torn, @blocks, @first_damaged, @resume_offset = Ext.stream(@buf)
|
|
22
40
|
end
|
|
23
41
|
|
|
24
|
-
def self.read(path
|
|
42
|
+
def self.read(path, continue_past_damage: false)
|
|
43
|
+
new(File.binread(path), continue_past_damage: continue_past_damage)
|
|
44
|
+
end
|
|
25
45
|
|
|
26
46
|
def torn? = @torn
|
|
27
47
|
|
|
48
|
+
def continue_past_damage? = @continue_past_damage
|
|
49
|
+
|
|
28
50
|
# The bytes up to and including the last commit.
|
|
29
51
|
def committed = @buf.byteslice(0, @committed_end)
|
|
30
52
|
|
|
@@ -47,13 +69,26 @@ module C0
|
|
|
47
69
|
|
|
48
70
|
alias each each_block
|
|
49
71
|
|
|
50
|
-
#
|
|
72
|
+
# Offset after block i's ETB and payload: where the next block starts.
|
|
73
|
+
def block_end(index) = index + 1 < @blocks.size ? @blocks[index + 1][0] : @committed_end
|
|
74
|
+
|
|
75
|
+
# Indexes of the damaged blocks.
|
|
51
76
|
def damaged = @blocks.each_index.select { |i| @blocks[i][2] }
|
|
52
77
|
|
|
53
78
|
def damaged?(index) = @blocks[index] ? @blocks[index][2] : false
|
|
54
79
|
|
|
55
|
-
#
|
|
56
|
-
def
|
|
80
|
+
# Whether any committed block is damaged.
|
|
81
|
+
def has_damaged? = !@first_damaged.nil?
|
|
82
|
+
|
|
83
|
+
# The committed region as a Container of documents, damaged blocks left
|
|
84
|
+
# out. Raises DamagedBlockError when a block is damaged, unless continuing.
|
|
85
|
+
def container
|
|
86
|
+
check_damage
|
|
87
|
+
Container.new(Ext.stream_undamaged(@buf))
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The first document of the committed region.
|
|
91
|
+
def document = container.first || Document.new("".b)
|
|
57
92
|
|
|
58
93
|
def body = document.body
|
|
59
94
|
|
|
@@ -63,10 +98,33 @@ module C0
|
|
|
63
98
|
b.groups.first || Group.new("".b, 0, 0, 0, 0, 0)
|
|
64
99
|
end
|
|
65
100
|
|
|
66
|
-
# Every committed record, across
|
|
67
|
-
|
|
101
|
+
# Every committed record in order, across documents and groups, up to
|
|
102
|
+
# the first damaged block; then DamagedBlockError, unless continuing, in
|
|
103
|
+
# which case every record of every sound block.
|
|
104
|
+
def each_record
|
|
105
|
+
return enum_for(:each_record) unless block_given?
|
|
106
|
+
|
|
107
|
+
if has_damaged? && !@continue_past_damage
|
|
108
|
+
yield_records(Container.new(Ext.stream_before_damage(@buf))) { |r| yield r }
|
|
109
|
+
raise DamagedBlockError.new(@first_damaged, @resume_offset)
|
|
110
|
+
end
|
|
111
|
+
yield_records(container) { |r| yield r }
|
|
112
|
+
nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def records = each_record.to_a
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
def yield_records(cont)
|
|
120
|
+
cont.documents.each { |doc| doc.body.groups.each { |g| g.records.each { |r| yield r } } }
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def check_damage
|
|
124
|
+
return if @continue_past_damage || !has_damaged?
|
|
68
125
|
|
|
69
|
-
|
|
126
|
+
raise DamagedBlockError.new(@first_damaged, @resume_offset)
|
|
127
|
+
end
|
|
70
128
|
end
|
|
71
129
|
|
|
72
130
|
# Position just past the last commit, found by a backward scan: a
|
data/lib/c0.rb
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
# is zero-copy: accessors return shared byteslices into the input buffer.
|
|
7
7
|
#
|
|
8
8
|
# buf = C0.build do |b|
|
|
9
|
-
# b.syn(
|
|
9
|
+
# b.syn(version: "1")
|
|
10
10
|
# b.group("users", %w[name amount]) do
|
|
11
11
|
# b.record("Alice", "1502.30")
|
|
12
12
|
# b.record("Bob", "340.00")
|
|
@@ -16,10 +16,14 @@
|
|
|
16
16
|
# doc = C0::Document.new(buf)
|
|
17
17
|
# doc["users"].record(0).value(0) # => "Alice"
|
|
18
18
|
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
19
|
+
# Parse is validate, as in the C core: constructing a Document, Container,
|
|
20
|
+
# Body or Stream::Reader applies the whole grammar (GRAMMAR.md) to the bytes
|
|
21
|
+
# and raises a C0::SyntaxError on malformed input; the navigators (Body,
|
|
22
|
+
# Group, Record) then assume well-formed input and report the result model,
|
|
23
|
+
# with dead elements dropped and sub-units (`␞a␟b␚␟c`) hanging off their
|
|
24
|
+
# units rather than counted among them. `C0.validate` is the same check
|
|
25
|
+
# on a whole buffer; `C0.canonical?` adds the canonical constraints and is
|
|
26
|
+
# opt-in, never a condition of reading.
|
|
23
27
|
|
|
24
28
|
begin
|
|
25
29
|
require "c0/c0_ext"
|
|
@@ -33,14 +37,14 @@ module C0
|
|
|
33
37
|
STX = 0x02 # Start of text (a literal value)
|
|
34
38
|
ETX = 0x03 # End of text
|
|
35
39
|
EOT = 0x04 # End of document
|
|
36
|
-
ENQ = 0x05 # Reference
|
|
40
|
+
ENQ = 0x05 # Reference: a group name, or a pattern (ENQ DLE … CAN)
|
|
37
41
|
BEL = 0x07 # Comment: a note, or marks the next element dead
|
|
38
42
|
DLE = 0x10 # Recurse: opens a nested level (outside text); escape (inside text)
|
|
39
43
|
SYN = 0x16 # Start of document, then the heading
|
|
40
44
|
ETB = 0x17 # Commit marker (stream mode)
|
|
41
45
|
CAN = 0x18 # Close a nested level
|
|
42
46
|
EM = 0x19 # End of medium: a raw blob follows
|
|
43
|
-
SUB = 0x1a #
|
|
47
|
+
SUB = 0x1a # Subordinate: the element after is a child of the element before (GS: subsection, US: sub-unit)
|
|
44
48
|
FS = 0x1c # File separator
|
|
45
49
|
GS = 0x1d # Group separator
|
|
46
50
|
RS = 0x1e # Record separator
|
|
@@ -86,14 +90,16 @@ module C0
|
|
|
86
90
|
# Layout trimmed from both ends of a raw slice.
|
|
87
91
|
def trim(raw) = Ext.trim(to_bin(raw))
|
|
88
92
|
|
|
89
|
-
# Check a whole buffer
|
|
90
|
-
# subclass) on the first error;
|
|
93
|
+
# Check a whole buffer (one or more documents, an optional EM tail) against
|
|
94
|
+
# the grammar. Raises a C0::SyntaxError (or a subclass) on the first error;
|
|
95
|
+
# returns nil when well-formed.
|
|
91
96
|
def validate(buf)
|
|
92
97
|
Ext.validate(to_bin(buf))
|
|
93
98
|
end
|
|
94
99
|
|
|
95
|
-
# Whether bytes are a canonical
|
|
96
|
-
# per value, unique keys in each keyed record, no framing, no
|
|
100
|
+
# Whether bytes are a canonical span: well-formed, no layout, one spelling
|
|
101
|
+
# per value, unique keys in each keyed record, no framing, no notes or dead
|
|
102
|
+
# elements. A SUB prefix is structure and is kept.
|
|
97
103
|
def canonical?(buf) = Ext.canonical(to_bin(buf))
|
|
98
104
|
|
|
99
105
|
# The logical value of a raw unit: a plain value trimmed, or the inside of a
|
|
@@ -121,3 +127,4 @@ require_relative "c0/document"
|
|
|
121
127
|
require_relative "c0/builder"
|
|
122
128
|
require_relative "c0/stream"
|
|
123
129
|
require_relative "c0/pretty"
|
|
130
|
+
require_relative "c0/diff"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: c0data
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Thomas Sawyer
|
|
@@ -21,11 +21,16 @@ files:
|
|
|
21
21
|
- LICENSE
|
|
22
22
|
- README.md
|
|
23
23
|
- c0-c/c0.h
|
|
24
|
+
- c0-c/c0_diff.c
|
|
25
|
+
- c0-c/c0_diff.h
|
|
26
|
+
- c0-c/c0_strbuf.h
|
|
27
|
+
- ext/c0/c0_diff_impl.c
|
|
24
28
|
- ext/c0/c0_ext.c
|
|
25
29
|
- ext/c0/extconf.rb
|
|
26
30
|
- lib/c0.rb
|
|
27
31
|
- lib/c0/body.rb
|
|
28
32
|
- lib/c0/builder.rb
|
|
33
|
+
- lib/c0/diff.rb
|
|
29
34
|
- lib/c0/document.rb
|
|
30
35
|
- lib/c0/pretty.rb
|
|
31
36
|
- lib/c0/stream.rb
|