c0data 0.3.1

data/ext/c0/extconf.rb

@@ -0,0 +1,7 @@
+require "mkmf"
+
+# c0.h lives in the c0-c submodule at the repository root.
+$INCFLAGS << " -I#{File.expand_path('../../c0-c', __dir__)}"
+$CFLAGS << " -Wall -Wextra"
+
+create_makefile("c0/c0_ext")

data/lib/c0/body.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+module C0
+  # Zero-copy navigator for a document body: the bytes between a heading and
+  # EOT, or the inside of a nested level. Everything it returns is a shared
+  # byteslice of the original buffer.
+  #
+  # Records before the first GS of a file form an unnamed group; records and
+  # groups before the first FS form an unnamed file. Enumerates the groups of
+  # the first file.
+  class Body
+    include Enumerable
+
+    attr_reader :buf, :start, :stop
+
+    def initialize(buf, start = 0, stop = nil)
+      @buf = C0.to_bin(buf)
+      @start = start
+      @stop = stop || @buf.bytesize
+    end
+
+    # The files, in order (at least one: the unnamed leading file).
+    def files
+      @files ||= Ext.files(@buf, @start, @stop).map { |ns, ne, s, e| FileView.new(@buf, ns, ne, s, e) }
+    end
+
+    # All groups of all files, in order.
+    def groups
+      @groups ||= Ext.groups(@buf, @start, @stop).map { |d, ns, ne, s, e| Group.new(@buf, d, ns, ne, s, e) }
+    end
+
+    # Groups of the first file.
+    def group_count = first_file.groups.size
+
+    # Group by index or by name (first file). Raises C0::KeyError for a name
+    # with no group.
+    def group(index_or_name)
+      index_or_name.is_a?(Integer) ? first_file.groups[index_or_name] : first_file.group(index_or_name)
+    end
+
+    alias [] group
+
+    def each_group(&) = first_file.groups.each(&)
+
+    alias each each_group
+
+    def group_names = first_file.groups.map(&:name)
+
+    # Name of the first file (empty when the body has no FS).
+    def name = first_file.name
+
+    # Records of the body when it is a bare record stream (no FS, no GS): the
+    # unnamed group's records.
+    def records = first_file.groups.first&.records || []
+
+    def record_count = records.size
+
+    def record(index) = records[index]
+
+    def each_record(&) = records.each(&)
+
+    # Whether the body is just records: one unnamed file, at most the unnamed group.
+    def bare? = Ext.body_bare(@buf, @start, @stop)
+
+    # The body's bytes.
+    def raw = @buf.byteslice(@start, @stop - @start)
+
+    private
+
+    def first_file = files[0]
+  end
+
+  # A file within a body: its name and groups. Enumerates the groups.
+  class FileView
+    include Enumerable
+
+    attr_reader :start, :stop
+
+    def initialize(buf, name_start, name_end, start, stop)
+      @buf = buf
+      @name_start = name_start
+      @name_end = name_end
+      @start = start
+      @stop = stop
+    end
+
+    def name = @buf.byteslice(@name_start, @name_end - @name_start)
+
+    def groups
+      @groups ||= Ext.groups(@buf, @start, @stop).map { |d, ns, ne, s, e| Group.new(@buf, d, ns, ne, s, e) }
+    end
+
+    # The first group with this name. Raises C0::KeyError when there is none.
+    def group(name)
+      needle = C0.to_bin(name)
+      groups.find { |g| g.name == needle } or raise KeyError, "No group named '#{name}'"
+    end
+
+    alias [] group
+
+    def each(&) = groups.each(&)
+
+    def raw = @buf.byteslice(@start, @stop - @start)
+  end
+
+  # A group: its depth (GS count; 0 for the unnamed leading group), name,
+  # optional header row and records. Enumerates the records.
+  class Group
+    include Enumerable
+
+    attr_reader :depth, :start, :stop
+
+    def initialize(buf, depth, name_start, name_end, start, stop)
+      @buf = buf
+      @depth = depth
+      @name_start = name_start
+      @name_end = name_end
+      @start = start
+      @stop = stop
+    end
+
+    def name = @buf.byteslice(@name_start, @name_end - @name_start)
+
+    # The header row as a Record, or nil.
+    def header_row
+      return @header_row if defined?(@header_row)
+
+      range = Ext.group_header(@buf, @start, @stop)
+      @header_row = range && Record.new(@buf, range[0], range[1])
+    end
+
+    def header? = !header_row.nil?
+
+    alias has_header? header?
+
+    # Header names, or an empty array.
+    def headers = header_row ? header_row.values : []
+
+    def header_count = header_row ? header_row.field_count : 0
+
+    def header(index) = headers[index]
+
+    # Index of a header name, or nil.
+    def column(name)
+      needle = C0.to_bin(name)
+      headers.index(needle)
+    end
+
+    # The live records (header row, notes, commits and dead elements skipped).
+    def records
+      @records ||= Ext.group_records(@buf, @start, @stop).map { |s, e| Record.new(@buf, s, e) }
+    end
+
+    def record_count = records.size
+
+    alias size record_count
+
+    def record(index) = records[index]
+
+    def each_record(&) = records.each(&)
+
+    alias each each_record
+
+    # The group as a table is itself.
+    def table = self
+
+    # The group's contents (after its name).
+    def raw = @buf.byteslice(@start, @stop - @start)
+  end
+
+  Table = Group
+
+  # A record: positional or keyed, never mixed. Zero-copy accessors.
+  # Enumerates the logical values of a positional record.
+  class Record
+    include Enumerable
+
+    attr_reader :start, :stop
+
+    def initialize(buf, start, stop)
+      @buf = buf
+      @start = start
+      @stop = stop
+    end
+
+    # The record's bytes (after its RS, trailing layout excluded).
+    def raw = @buf.byteslice(@start, @stop - @start)
+
+    # Whether the record is keyed (its first live unit is an SOH).
+    def keyed? = Ext.record_keyed(@buf, @start, @stop)
+
+    # ---- positional ---------------------------------------------------------
+
+    # Raw bytes of unit n (its layout included), or of the unit under a key
+    # (last wins); nil when there is none.
+    def field(index_or_key)
+      s, e = index_or_key.is_a?(Integer) ? units[index_or_key] : keyed_range(index_or_key)
+      s && @buf.byteslice(s, e - s)
+    end
+
+    # Number of positional units (0 for a keyed record).
+    def field_count = units.size
+
+    alias size field_count
+
+    def fields = units.map { |s, e| @buf.byteslice(s, e - s) }
+
+    # Logical value of unit n, or nil when there is none.
+    def value(index)
+      s, e = units[index]
+      s && Ext.value(@buf, s, e)
+    end
+
+    def values = units.map { |s, e| Ext.value(@buf, s, e) }
+
+    def each(&) = values.each(&)
+
+    # Unit n (or the unit under a key) as a nested level. Raises C0::Error when
+    # it is not one; nil for a key with no unit.
+    def level(index_or_key)
+      s, e = index_or_key.is_a?(Integer) ? units[index_or_key] : keyed_range(index_or_key)
+      return nil if s.nil? && !index_or_key.is_a?(Integer)
+      raise KeyError, "no unit #{index_or_key}" if s.nil?
+
+      ls, le = Ext.unit_level(@buf, s, e)
+      Body.new(@buf, ls, le)
+    end
+
+    # Unit n as a flat list: a level holding at most one positional record.
+    # Raises C0::Error for any other level.
+    def list(index)
+      s, e = units[index]
+      raise KeyError, "no unit #{index}" if s.nil?
+
+      Ext.unit_list(@buf, s, e).map { |is, ie| Ext.value(@buf, is, ie) }
+    end
+
+    # Unit n as a Reference.
+    def ref(index)
+      s, e = units[index]
+      raise KeyError, "no unit #{index}" if s.nil?
+
+      Reference.new(Ext.unit_ref(@buf, s, e).map { |rs, re| @buf.byteslice(rs, re - rs) })
+    end
+
+    # ---- keyed --------------------------------------------------------------
+
+    # Keys in written order (duplicates included).
+    def keys = pair_ranges.map { |ks, ke, _, _| @buf.byteslice(ks, ke - ks) }
+
+    # [key, logical value] pairs in written order.
+    def pairs = pair_ranges.map { |ks, ke, vs, ve| [@buf.byteslice(ks, ke - ks), Ext.value(@buf, vs, ve)] }
+
+    # Logical value under a key (last wins), or nil.
+    def [](key)
+      s, e = keyed_range(key)
+      s && Ext.value(@buf, s, e)
+    end
+
+    def key?(key) = !keyed_range(key).nil?
+
+    alias has_key? key?
+
+    # The keyed record as a Hash (last wins).
+    def to_h = pairs.to_h
+
+    private
+
+    def units = @units ||= Ext.record_units(@buf, @start, @stop)
+
+    def pair_ranges = @pair_ranges ||= Ext.record_pairs(@buf, @start, @stop)
+
+    def keyed_range(key)
+      needle = C0.to_bin(key)
+      found = nil
+      pair_ranges.each do |ks, ke, vs, ve|
+        found = [vs, ve] if @buf.byteslice(ks, ke - ks) == needle
+      end
+      found
+    end
+  end
+
+  # A reference: a group name, or a path of two or three segments
+  # (group, record id, field name).
+  class Reference
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    # Parse a raw unit (ENQ …). Raises C0::Error when it is not a reference.
+    def self.parse(raw)
+      b = C0.to_bin(raw)
+      new(Ext.unit_ref(b, 0, b.bytesize).map { |s, e| b.byteslice(s, e - s) })
+    end
+
+    def group = @path[0]
+
+    def record = @path[1]
+
+    def field = @path[2]
+
+    def group? = @path.size == 1
+
+    def ==(other) = other.is_a?(Reference) && other.path == @path
+
+    alias eql? ==
+
+    def hash = @path.hash
+
+    def to_s = @path.join("/")
+  end
+end

data/lib/c0/builder.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module C0
+  # Writes C0DATA in canonical compact form. The class and its methods are
+  # defined in the C extension (a TypedData object owning a c0_builder); this
+  # file adds the Ruby conveniences.
+  #
+  #   buf = C0::Builder.build do |b|
+  #     b.syn(shape: "tabular")
+  #     b.group("users", %w[name amount]) do
+  #       b.record("Alice", "1502.30")
+  #       b.record("Bob", "340.00")
+  #     end
+  #     b.eot
+  #   end
+  #
+  # Two layers. The primitives are one method per code (`fs`, `gs`, `rs`,
+  # `us`, `sub`, `soh`, `dle`, `can`, `bel`, `enq`, `syn`, `eot`, `etb`, `em`);
+  # each writes its code and, when given, what grammatically follows it, so a
+  # chain reads like the bytes (`sub` is C0-DIFF only: `rs("old").sub("new")`
+  # writes a replace-all, ␞old␚new; in a data body SUB is an error):
+  #
+  #   b.rs("Alice").us("1502.30")                              # ␞Alice␟1502.30
+  #   b.rs.soh("host").us("localhost").soh("port").us("5432")  # ␞␁host␟localhost␁port␟5432
+  #   b.gs("users").soh.rs("name").us("amount")                # ␝users␁␞name␟amount
+  #   b.bel.gs("old")                                          # ␇␝old, a dead group
+  #
+  # The composed methods (`file`, `group`, `header`, `record`, `field`, `map`,
+  # `key`, `level`, `list`, `ref`, `note`) are built on the primitives and
+  # cannot write ill-formed output. The primitives write what you spell;
+  # well-formedness is then yours to keep. Both layers share the record state,
+  # so they mix: `b.rs("a").field("b")` writes ␞a␟b.
+  #
+  # Values are byte-transparent and written in their one spelling: plain when
+  # they can be, text (STX … ETX) otherwise, with only the seven codes escaped;
+  # there is no STX/ETX primitive, so everything written stays canonical.
+  # Names (file, group, header, key, reference segment, ETB payload) must be
+  # plain, or a C0::BuildError is raised before anything is written. Every
+  # method returns the builder, so calls chain.
+  #
+  # A record is opened by `record` or `rs` (or implicitly by the first `field`,
+  # `key`, `level`, `list` or `ref`) and ended by the next structural call.
+  # Keyed units written right after `syn` belong to the heading. Inside
+  # `level { }` (or between `dle` and `can`) the builder is at the top of a
+  # fresh body.
+  class Builder
+    # Drive a fresh builder and return its bytes.
+    def self.build
+      b = new
+      yield b
+      b.bytes
+    end
+
+    # Alias of `bytes`: the compact form as a binary String.
+    def to_s = bytes
+
+    alias to_slice bytes
+
+    # Compatibility spelling of `list`.
+    def list_field(items) = list(items)
+
+    def inspect = "#<C0::Builder #{bytesize} bytes, depth #{depth}>"
+  end
+end

data/lib/c0/document.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module C0
+  # The heading of a document: the keyed units after SYN. Last wins for a
+  # duplicate key. Enumerates [key, value] pairs.
+  class Heading
+    include Enumerable
+
+    KNOWN_KEYS = %w[shape version blob].freeze
+
+    def initialize(buf, start, stop)
+      @buf = buf
+      @start = start
+      @stop = stop
+    end
+
+    # [key, value] in written order: the logical value, or for a nested level
+    # (the blob table) its raw bytes.
+    def pairs
+      @pairs ||= raw_pairs.map do |k, s, e|
+        [k, Ext.unit_type(@buf, s, e) == UnitType::LEVEL ? @buf.byteslice(s, e - s) : Ext.value(@buf, s, e)]
+      end
+    end
+
+    def each(&) = pairs.each(&)
+
+    def keys = raw_pairs.map(&:first)
+
+    # Value for a key (last wins), or nil.
+    def [](key)
+      needle = C0.to_bin(key)
+      found = nil
+      pairs.each { |k, v| found = v if k == needle }
+      found
+    end
+
+    def key?(key) = !self[key].nil?
+
+    def empty? = raw_pairs.empty?
+
+    # The `shape` value as a String, or nil.
+    def shape = self["shape"]&.dup&.force_encoding(Encoding::UTF_8)
+
+    # The `version` value as an Integer; 1 when absent or not a number.
+    def version
+      v = self["version"]
+      v && v.match?(/\A\d+\z/) ? v.to_i : 1
+    end
+
+    # The value under a key as a nested level, or nil when absent or not one.
+    def level(key)
+      range = raw_range(key)
+      return nil unless range && Ext.unit_type(@buf, *range) == UnitType::LEVEL
+
+      ls, le = Ext.unit_level(@buf, *range)
+      Body.new(@buf, ls, le)
+    end
+
+    # The blob table as a Body, when declared as a level.
+    def blob = level("blob")
+
+    # The heading's bytes.
+    def raw = @buf.byteslice(@start, @stop - @start)
+
+    private
+
+    def raw_pairs
+      @raw_pairs ||= Ext.record_pairs(@buf, @start, @stop).map { |ks, ke, vs, ve| [@buf.byteslice(ks, ke - ks), vs, ve] }
+    end
+
+    def raw_range(key)
+      needle = C0.to_bin(key)
+      found = nil
+      raw_pairs.each { |k, s, e| found = [s, e] if k == needle }
+      found
+    end
+  end
+
+  # One C0DATA document: an optional SYN heading, a body, an optional EOT.
+  # `Document.new(buf)` reads the first document in the buffer. Use
+  # `Container` for a buffer holding several documents or an EM tail.
+  # Enumerates the groups of the body's first file.
+  class Document
+    include Enumerable
+
+    SHAPES = %w[diff stream tabular document key-value].freeze
+
+    attr_reader :buf, :heading, :body, :start, :stop
+
+    def initialize(buf, start = 0)
+      @buf = C0.to_bin(buf)
+      load(Ext.document(@buf, start))
+    end
+
+    # A document from the C reader's description (see Container).
+    def self.from_ext(buf, desc)
+      doc = allocate
+      doc.instance_variable_set(:@buf, buf)
+      doc.send(:load, desc)
+      doc
+    end
+
+    def has_syn? = @has_syn
+
+    alias syn? has_syn?
+
+    def has_eot? = @has_eot
+
+    alias eot? has_eot?
+
+    # Loud warnings (DESIGN.md "Warnings") as strings; empty for a careful writer's file.
+    def warnings
+      @warnings ||= begin
+        w = []
+        w << "no SYN at the start of the document" unless @has_syn
+        w << "no EOT at the end of the document" if !@has_eot && heading.shape != "stream"
+        heading.keys.each do |k|
+          w << "unknown heading key '#{k}'" unless Heading::KNOWN_KEYS.include?(k)
+        end
+        seen = {}
+        heading.keys.each { |k| w << "duplicate heading key '#{k}'" if seen.key?(k) && !w.include?("duplicate heading key '#{k}'"); seen[k] = true }
+        if (sh = heading.shape) && !SHAPES.include?(sh)
+          w << "unknown shape '#{sh}'"
+        end
+        if (v = heading["version"]) && v != "1".b
+          w << "unknown version '#{v}'"
+        end
+        w
+      end
+    end
+
+    # ---- convenience accessors over the body --------------------------------
+
+    def name = @body.name
+
+    def files = @body.files
+
+    def group_count = @body.group_count
+
+    def group(index_or_name) = @body.group(index_or_name)
+
+    alias [] group
+
+    def each_group(&) = @body.each_group(&)
+
+    alias each each_group
+
+    def groups = @body.groups
+
+    def group_names = @body.group_names
+
+    def records = @body.records
+
+    def shape = heading.shape
+
+    # The document's bytes (SYN through EOT).
+    def raw = @buf.byteslice(@start, @stop - @start)
+
+    private
+
+    def load(desc)
+      @start, @stop, @has_syn, @has_eot, hs, he, bs, be, _bits = desc
+      @heading = Heading.new(@buf, hs, he)
+      @body = Body.new(@buf, bs, be)
+    end
+  end
+
+  # A file or message: one or more documents and an optional raw tail after
+  # EM. Enumerates the documents.
+  class Container
+    include Enumerable
+
+    attr_reader :buf, :documents, :tail
+
+    def initialize(buf)
+      @buf = C0.to_bin(buf)
+      descs, @has_em, em_pos = Ext.documents(@buf)
+      @documents = descs.map { |d| Document.from_ext(@buf, d) }
+      @tail = @has_em ? @buf.byteslice(em_pos + 1, @buf.bytesize - em_pos - 1) : "".b
+    end
+
+    def has_em? = @has_em
+
+    alias em? has_em?
+
+    def each(&) = @documents.each(&)
+
+    def first = @documents[0]
+
+    def size = @documents.size
+
+    # Loud warnings about the container as a whole; each document carries its own.
+    def warnings
+      @warnings ||= begin
+        w = []
+        @documents.each_with_index do |doc, i|
+          next unless doc.heading.key?("blob")
+
+          if i.positive?
+            w << "blob table in a later heading is ignored"
+          elsif doc.heading.blob.nil?
+            w << "blob is not a level: declares nothing"
+          elsif !doc.heading.blob.groups.first&.header?
+            w << "blob table has no header row: declares nothing"
+          end
+        end
+        w << "EM with no blob table: the region is unaddressable" if @has_em && blob_table.nil?
+        w
+      end
+    end
+
+    # The blob table from the first document's heading, as a Body, or nil.
+    def blob_table = first&.heading&.blob
+
+    # Bytes of a named blob from the tail (columns matched by header name,
+    # last row wins), or nil. With no name, the blob table itself.
+    def blob(name = nil)
+      return blob_table if name.nil?
+
+      table = blob_table or return nil
+      g = table.groups.first or return nil
+      ni = g.column("name") || 0
+      oi = g.column("offset") || 1
+      li = g.column("length") || 2
+      needle = C0.to_bin(name)
+      found = nil
+      g.records.each do |r|
+        next unless r.value(ni) == needle
+
+        off = r.value(oi)
+        len = r.value(li)
+        next unless off&.match?(/\A\d+\z/) && len&.match?(/\A\d+\z/)
+
+        off = off.to_i
+        len = len.to_i
+        next if off > @tail.bytesize || len > @tail.bytesize - off
+
+        found = @tail.byteslice(off, len)
+      end
+      found
+    end
+  end
+end

data/lib/c0/pretty.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module C0
+  # Pretty form: Unicode Control Pictures (U+2400 block) for the codes, line
+  # breaks and indentation as layout. `format` and `parse` are inverses on
+  # well-formed input. GlyphInDataError (a C0::Error) is raised when a value
+  # holds a Control Picture character and cannot be shown unambiguously.
+  module Pretty
+    MODES = { compact: 0, aligned: 1, spaced: 2 }.freeze
+
+    module_function
+
+    # The glyph for a code byte.
+    def glyph(byte) = (0x2400 + byte).chr(Encoding::UTF_8)
+
+    # Compact bytes → pretty form (a UTF-8 String). `mode` is :compact (one
+    # element per line), :aligned (table columns aligned) or :spaced (aligned,
+    # plus a space after each prefix and around the US glyph).
+    def format(buf, indent: "  ", mode: :compact)
+      m = MODES.fetch(mode) { raise ArgumentError, "unknown mode #{mode.inspect}; use #{MODES.keys.inspect}" }
+      Ext.pretty_format(C0.to_bin(buf), indent, m)
+    end
+
+    # Pretty form → compact bytes. Glyphs become codes; outside text,
+    # whitespace touching a glyph or a line edge is dropped.
+    def parse(text) = Ext.pretty_parse(text.to_s.encode(Encoding::UTF_8))
+  end
+end