c0data 0.3.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/c0/document.rb CHANGED
@@ -1,12 +1,15 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module C0
4
- # The heading of a document: the keyed units after SYN. Last wins for a
5
- # duplicate key. Enumerates [key, value] pairs.
4
+ # The heading of a document: the keyed units after SYN (dead units
5
+ # skipped; the heading's values take no sub-units). Last wins for a duplicate key. Enumerates [key, value] pairs.
6
6
  class Heading
7
7
  include Enumerable
8
8
 
9
- KNOWN_KEYS = %w[shape version blob].freeze
9
+ # The keys the format defines (DESIGN.md "Heading"): `shape` is `diff` or
10
+ # `stream`; `schema` a name, or a level holding one keyed record of
11
+ # `name`, `url`, `hash`; `version`; `blob` the blob table.
12
+ KNOWN_KEYS = %w[shape schema version blob].freeze
10
13
 
11
14
  def initialize(buf, start, stop)
12
15
  @buf = buf
@@ -15,7 +18,7 @@ module C0
15
18
  end
16
19
 
17
20
  # [key, value] in written order: the logical value, or for a nested level
18
- # (the blob table) its raw bytes.
21
+ # (the blob table, a schema record) its raw bytes.
19
22
  def pairs
20
23
  @pairs ||= raw_pairs.map do |k, s, e|
21
24
  [k, Ext.unit_type(@buf, s, e) == UnitType::LEVEL ? @buf.byteslice(s, e - s) : Ext.value(@buf, s, e)]
@@ -47,13 +50,19 @@ module C0
47
50
  v && v.match?(/\A\d+\z/) ? v.to_i : 1
48
51
  end
49
52
 
53
+ # The schema the body claims: a plain name (bytes), or for the keyed form
54
+ # the one keyed Record of `name`, `url`, `hash`; nil when absent. The
55
+ # format checks the form; a schema layer checks the match. The same
56
+ # shape in every port.
57
+ def schema = level("schema")&.record(0) || self["schema"]
58
+
50
59
  # The value under a key as a nested level, or nil when absent or not one.
51
60
  def level(key)
52
61
  range = raw_range(key)
53
62
  return nil unless range && Ext.unit_type(@buf, *range) == UnitType::LEVEL
54
63
 
55
64
  ls, le = Ext.unit_level(@buf, *range)
56
- Body.new(@buf, ls, le)
65
+ Body.view(@buf, ls, le)
57
66
  end
58
67
 
59
68
  # The blob table as a Body, when declared as a level.
@@ -77,15 +86,25 @@ module C0
77
86
  end
78
87
 
79
88
  # One C0DATA document: an optional SYN heading, a body, an optional EOT.
80
- # `Document.new(buf)` reads the first document in the buffer. Use
81
- # `Container` for a buffer holding several documents or an EM tail.
82
- # Enumerates the groups of the body's first file.
89
+ # `Document.new(buf)` parses the first document in the buffer, applying the
90
+ # whole grammar (a C0::SyntaxError on malformed input). Use `Container` for
91
+ # a buffer holding several documents or an EM tail. Enumerates the groups
92
+ # of the body's first file.
83
93
  class Document
84
94
  include Enumerable
85
95
 
86
- SHAPES = %w[diff stream tabular document key-value].freeze
96
+ # The shapes the format defines; any other is a loud warning.
97
+ SHAPES = %w[diff stream].freeze
98
+
99
+ # Warning bits, as the C core reports them (DESIGN.md "Warnings").
100
+ WARN_NO_SYN = 0x01
101
+ WARN_NO_EOT = 0x02
102
+ WARN_UNKNOWN_KEY = 0x04
103
+ WARN_DUPLICATE_KEY = 0x08
104
+ WARN_UNKNOWN_SHAPE = 0x10
105
+ WARN_UNKNOWN_VERSION = 0x20
87
106
 
88
- attr_reader :buf, :heading, :body, :start, :stop
107
+ attr_reader :buf, :heading, :body, :start, :stop, :warning_bits, :dead_elements
89
108
 
90
109
  def initialize(buf, start = 0)
91
110
  @buf = C0.to_bin(buf)
@@ -108,27 +127,29 @@ module C0
108
127
 
109
128
  alias eot? has_eot?
110
129
 
111
- # Loud warnings (DESIGN.md "Warnings") as strings; empty for a careful writer's file.
130
+ # Loud warnings (DESIGN.md "Warnings") as strings; empty for a careful
131
+ # writer's file. Warnings never fail a parse.
112
132
  def warnings
113
133
  @warnings ||= begin
114
134
  w = []
115
- w << "no SYN at the start of the document" unless @has_syn
116
- w << "no EOT at the end of the document" if !@has_eot && heading.shape != "stream"
117
- heading.keys.each do |k|
118
- w << "unknown heading key '#{k}'" unless Heading::KNOWN_KEYS.include?(k)
119
- end
120
- seen = {}
121
- heading.keys.each { |k| w << "duplicate heading key '#{k}'" if seen.key?(k) && !w.include?("duplicate heading key '#{k}'"); seen[k] = true }
122
- if (sh = heading.shape) && !SHAPES.include?(sh)
123
- w << "unknown shape '#{sh}'"
135
+ w << "no SYN at the start of the document" if warn?(WARN_NO_SYN)
136
+ w << "no EOT at the end of the document" if warn?(WARN_NO_EOT)
137
+ if warn?(WARN_UNKNOWN_KEY)
138
+ heading.keys.uniq.each { |k| w << "unknown heading key '#{k}'" unless Heading::KNOWN_KEYS.include?(k) }
124
139
  end
125
- if (v = heading["version"]) && v != "1".b
126
- w << "unknown version '#{v}'"
140
+ if warn?(WARN_DUPLICATE_KEY)
141
+ heading.keys.tally.each { |k, n| w << "duplicate heading key '#{k}'" if n > 1 }
127
142
  end
143
+ w << "unknown shape '#{heading.shape}'" if warn?(WARN_UNKNOWN_SHAPE)
144
+ w << "unknown version '#{heading['version']}'" if warn?(WARN_UNKNOWN_VERSION)
128
145
  w
129
146
  end
130
147
  end
131
148
 
149
+ # Whether the body held any dead element (a dead record, header row,
150
+ # group or file); they are not in the model.
151
+ def dead_elements? = @dead_elements.positive?
152
+
132
153
  # ---- convenience accessors over the body --------------------------------
133
154
 
134
155
  def name = @body.name
@@ -158,15 +179,18 @@ module C0
158
179
 
159
180
  private
160
181
 
182
+ def warn?(bit) = (@warning_bits & bit) != 0
183
+
161
184
  def load(desc)
162
- @start, @stop, @has_syn, @has_eot, hs, he, bs, be, _bits = desc
185
+ @start, @stop, @has_syn, @has_eot, hs, he, bs, be, @warning_bits, @dead_elements = desc
163
186
  @heading = Heading.new(@buf, hs, he)
164
- @body = Body.new(@buf, bs, be)
187
+ @body = Body.view(@buf, bs, be)
165
188
  end
166
189
  end
167
190
 
168
191
  # A file or message: one or more documents and an optional raw tail after
169
- # EM. Enumerates the documents.
192
+ # EM. Parsing applies the whole grammar to every document. Enumerates the
193
+ # documents.
170
194
  class Container
171
195
  include Enumerable
172
196
 
data/lib/c0/stream.rb CHANGED
@@ -7,24 +7,46 @@ module C0
7
7
  # r.torn? # => data trails the last commit
8
8
  # r.each_record { |rec| ... }
9
9
  module Stream
10
+ # A committed block that is not well-formed (DESIGN.md "Damaged Blocks").
11
+ # `resume_offset` is where reading can go on: after the block's ETB and
12
+ # payload.
13
+ class DamagedBlockError < C0::Error
14
+ attr_reader :index, :resume_offset
15
+
16
+ def initialize(index, resume_offset)
17
+ @index = index
18
+ @resume_offset = resume_offset
19
+ super("block #{index} is damaged; reading can resume at byte #{resume_offset}")
20
+ end
21
+ end
22
+
10
23
  # Scans a buffer for commits. An unescaped ETB is always a commit, even
11
- # inside open text or a level; such a block is damaged and yields nothing.
12
- # The tail after the last commit is torn unless it holds only layout,
13
- # notes, EOT or EM. Enumerates the committed blocks' bytes.
24
+ # inside open text or a level. A committed block that is not well-formed,
25
+ # cut by its ETB or holding any other error, is **damaged**: it delivers
26
+ # nothing and adds nothing to the parse state. By default the reader
27
+ # delivers the records before the first damaged block and then raises
28
+ # DamagedBlockError; with `continue_past_damage: true` it skips the block
29
+ # and reads on. The tail after the last commit is torn unless it holds
30
+ # only layout, notes, EOT or EM. Enumerates the committed blocks' bytes.
14
31
  class Reader
15
32
  include Enumerable
16
33
 
17
- attr_reader :buf, :committed_end
34
+ attr_reader :buf, :committed_end, :first_damaged, :resume_offset
18
35
 
19
- def initialize(buf)
36
+ def initialize(buf, continue_past_damage: false)
20
37
  @buf = C0.to_bin(buf)
21
- @committed_end, @torn, @blocks = Ext.stream(@buf)
38
+ @continue_past_damage = continue_past_damage
39
+ @committed_end, @torn, @blocks, @first_damaged, @resume_offset = Ext.stream(@buf)
22
40
  end
23
41
 
24
- def self.read(path) = new(File.binread(path))
42
+ def self.read(path, continue_past_damage: false)
43
+ new(File.binread(path), continue_past_damage: continue_past_damage)
44
+ end
25
45
 
26
46
  def torn? = @torn
27
47
 
48
+ def continue_past_damage? = @continue_past_damage
49
+
28
50
  # The bytes up to and including the last commit.
29
51
  def committed = @buf.byteslice(0, @committed_end)
30
52
 
@@ -47,13 +69,26 @@ module C0
47
69
 
48
70
  alias each each_block
49
71
 
50
- # Indexes of the damaged blocks (cut by an ETB inside open text or a level).
72
+ # Offset after block i's ETB and payload: where the next block starts.
73
+ def block_end(index) = index + 1 < @blocks.size ? @blocks[index + 1][0] : @committed_end
74
+
75
+ # Indexes of the damaged blocks.
51
76
  def damaged = @blocks.each_index.select { |i| @blocks[i][2] }
52
77
 
53
78
  def damaged?(index) = @blocks[index] ? @blocks[index][2] : false
54
79
 
55
- # The committed region as a document, undamaged blocks only.
56
- def document = Document.new(Ext.stream_undamaged(@buf))
80
+ # Whether any committed block is damaged.
81
+ def has_damaged? = !@first_damaged.nil?
82
+
83
+ # The committed region as a Container of documents, damaged blocks left
84
+ # out. Raises DamagedBlockError when a block is damaged, unless continuing.
85
+ def container
86
+ check_damage
87
+ Container.new(Ext.stream_undamaged(@buf))
88
+ end
89
+
90
+ # The first document of the committed region.
91
+ def document = container.first || Document.new("".b)
57
92
 
58
93
  def body = document.body
59
94
 
@@ -63,10 +98,33 @@ module C0
63
98
  b.groups.first || Group.new("".b, 0, 0, 0, 0, 0)
64
99
  end
65
100
 
66
- # Every committed record, across all groups.
67
- def records = body.groups.flat_map(&:records)
101
+ # Every committed record in order, across documents and groups, up to
102
+ # the first damaged block; then DamagedBlockError, unless continuing, in
103
+ # which case every record of every sound block.
104
+ def each_record
105
+ return enum_for(:each_record) unless block_given?
106
+
107
+ if has_damaged? && !@continue_past_damage
108
+ yield_records(Container.new(Ext.stream_before_damage(@buf))) { |r| yield r }
109
+ raise DamagedBlockError.new(@first_damaged, @resume_offset)
110
+ end
111
+ yield_records(container) { |r| yield r }
112
+ nil
113
+ end
114
+
115
+ def records = each_record.to_a
116
+
117
+ private
118
+
119
+ def yield_records(cont)
120
+ cont.documents.each { |doc| doc.body.groups.each { |g| g.records.each { |r| yield r } } }
121
+ end
122
+
123
+ def check_damage
124
+ return if @continue_past_damage || !has_damaged?
68
125
 
69
- def each_record(&) = records.each(&)
126
+ raise DamagedBlockError.new(@first_damaged, @resume_offset)
127
+ end
70
128
  end
71
129
 
72
130
  # Position just past the last commit, found by a backward scan: a
data/lib/c0.rb CHANGED
@@ -6,7 +6,7 @@
6
6
  # is zero-copy: accessors return shared byteslices into the input buffer.
7
7
  #
8
8
  # buf = C0.build do |b|
9
- # b.syn(shape: "tabular")
9
+ # b.syn(version: "1")
10
10
  # b.group("users", %w[name amount]) do
11
11
  # b.record("Alice", "1502.30")
12
12
  # b.record("Bob", "340.00")
@@ -16,10 +16,14 @@
16
16
  # doc = C0::Document.new(buf)
17
17
  # doc["users"].record(0).value(0) # => "Alice"
18
18
  #
19
- # Reading is a two-step contract, as in the C core: `C0.validate` checks a
20
- # buffer against the grammar once; the navigators (Document, Body, Group,
21
- # Record) then assume well-formed input and raise only on the malformations
22
- # they happen to meet.
19
+ # Parse is validate, as in the C core: constructing a Document, Container,
20
+ # Body or Stream::Reader applies the whole grammar (GRAMMAR.md) to the bytes
21
+ # and raises a C0::SyntaxError on malformed input; the navigators (Body,
22
+ # Group, Record) then assume well-formed input and report the result model,
23
+ # with dead elements dropped and sub-units (`␞a␟b␚␟c`) hanging off their
24
+ # units rather than counted among them. `C0.validate` is the same check
25
+ # on a whole buffer; `C0.canonical?` adds the canonical constraints and is
26
+ # opt-in, never a condition of reading.
23
27
 
24
28
  begin
25
29
  require "c0/c0_ext"
@@ -33,14 +37,14 @@ module C0
33
37
  STX = 0x02 # Start of text (a literal value)
34
38
  ETX = 0x03 # End of text
35
39
  EOT = 0x04 # End of document
36
- ENQ = 0x05 # Reference
40
+ ENQ = 0x05 # Reference: a group name, or a pattern (ENQ DLE … CAN)
37
41
  BEL = 0x07 # Comment: a note, or marks the next element dead
38
42
  DLE = 0x10 # Recurse: opens a nested level (outside text); escape (inside text)
39
43
  SYN = 0x16 # Start of document, then the heading
40
44
  ETB = 0x17 # Commit marker (stream mode)
41
45
  CAN = 0x18 # Close a nested level
42
46
  EM = 0x19 # End of medium: a raw blob follows
43
- SUB = 0x1a # Substitution (C0-DIFF only)
47
+ SUB = 0x1a # Subordinate: the element after is a child of the element before (GS: subsection, US: sub-unit)
44
48
  FS = 0x1c # File separator
45
49
  GS = 0x1d # Group separator
46
50
  RS = 0x1e # Record separator
@@ -86,14 +90,16 @@ module C0
86
90
  # Layout trimmed from both ends of a raw slice.
87
91
  def trim(raw) = Ext.trim(to_bin(raw))
88
92
 
89
- # Check a whole buffer against the grammar. Raises a C0::SyntaxError (or a
90
- # subclass) on the first error; returns nil when well-formed.
93
+ # Check a whole buffer (one or more documents, an optional EM tail) against
94
+ # the grammar. Raises a C0::SyntaxError (or a subclass) on the first error;
95
+ # returns nil when well-formed.
91
96
  def validate(buf)
92
97
  Ext.validate(to_bin(buf))
93
98
  end
94
99
 
95
- # Whether bytes are a canonical unit: well-formed, no layout, one spelling
96
- # per value, unique keys in each keyed record, no framing, no comments.
100
+ # Whether bytes are a canonical span: well-formed, no layout, one spelling
101
+ # per value, unique keys in each keyed record, no framing, no notes or dead
102
+ # elements. A SUB prefix is structure and is kept.
97
103
  def canonical?(buf) = Ext.canonical(to_bin(buf))
98
104
 
99
105
  # The logical value of a raw unit: a plain value trimmed, or the inside of a
@@ -121,3 +127,4 @@ require_relative "c0/document"
121
127
  require_relative "c0/builder"
122
128
  require_relative "c0/stream"
123
129
  require_relative "c0/pretty"
130
+ require_relative "c0/diff"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: c0data
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Sawyer
@@ -21,11 +21,16 @@ files:
21
21
  - LICENSE
22
22
  - README.md
23
23
  - c0-c/c0.h
24
+ - c0-c/c0_diff.c
25
+ - c0-c/c0_diff.h
26
+ - c0-c/c0_strbuf.h
27
+ - ext/c0/c0_diff_impl.c
24
28
  - ext/c0/c0_ext.c
25
29
  - ext/c0/extconf.rb
26
30
  - lib/c0.rb
27
31
  - lib/c0/body.rb
28
32
  - lib/c0/builder.rb
33
+ - lib/c0/diff.rb
29
34
  - lib/c0/document.rb
30
35
  - lib/c0/pretty.rb
31
36
  - lib/c0/stream.rb