opencdd 0.1.1

data/lib/opencdd/cddal/lexer.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Opencdd
+  module Cddal
+    Token = Struct.new(:kind, :value, :line, :column, keyword_init: true) do
+      def inspect
+        "#{kind}#{value && !value.empty? ? "(#{value.inspect})" : ""}@#{line}:#{column}"
+      end
+    end
+
+    class Lexer
+      PUNCTUATION = {
+        "{" => :LBRACE, "}" => :RBRACE,
+        ":" => :COLON,  "," => :COMMA,
+        "<" => :LANGLE, ">" => :RANGLE,
+        "." => :DOT,
+        "(" => :LPAREN, ")" => :RPAREN,
+      }.freeze
+
+      KEYWORD_MAP = {
+        "instance" => :INSTANCE,
+        "alias"    => :ALIAS,
+        "import"   => :IMPORT,
+        "true"     => :TRUE,
+        "false"    => :FALSE,
+        "null"     => :NULL,
+      }.freeze
+
+      # Soft keywords. These are reserved only inside import
+      # declarations; elsewhere they're valid identifiers (e.g.
+      # `short_name.en: as` for attosecond). The lexer emits IDENT
+      # and the parser validates the value in import position.
+      SOFT_KEYWORDS = %w[as from].freeze
+
+      IRDI_RE         = /[0-9]+\/[0-9]+\/\/\/[A-Za-z0-9_]+(?:_[0-9]+)?#[A-Za-z0-9_]+/.freeze
+      LOCAL_REF_RE    = /[A-Za-z_][A-Za-z0-9_]*#[A-Za-z0-9_]+(?:##[A-Za-z0-9_]+)?/.freeze
+      DATE_RE         = /\d{4}-\d{2}-\d{2}/.freeze
+      NUMBER_RE       = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/.freeze
+      IDENT_RE        = /[A-Za-z_][A-Za-z0-9_]*/.freeze
+      META_CLASS_RE   = /(?:property-meta-class|enumeration-meta-class|term-meta-class|meta-class)(?![A-Za-z0-9_-])/.freeze
+      WS_RE           = /[ \t\r]+/.freeze
+      NEWLINE_RE      = /\n/.freeze
+      COMMENT_RE      = /#[^\n]*/.freeze
+
+      attr_reader :source
+
+      def initialize(source)
+        @source = source.to_s
+        @ss = StringScanner.new(@source)
+        @line = 1
+        @col = 1
+      end
+
+      def tokens
+        return @tokens if @tokens
+        @tokens = []
+        until @ss.eos?
+          skip_interstitial
+          break if @ss.eos?
+          tok = scan_token
+          raise LexError, "unexpected character #{@ss.peek(1).inspect} at line #{@line}, column #{@col}" unless tok
+          @tokens << tok
+        end
+        @tokens << Token.new(kind: :EOF, value: "", line: @line, column: @col)
+        @tokens
+      end
+
+      def next_token
+        tok = tokens[@pos || 0]
+        @pos = (@pos || 0) + 1
+        tok ? [tok.kind, tok.value] : [false, false]
+      end
+
+      private
+
+      def skip_interstitial
+        loop do
+          if ws = @ss.scan(WS_RE)
+            @col += ws.length
+          elsif @ss.scan(NEWLINE_RE)
+            @line += 1
+            @col = 1
+          elsif comment = @ss.scan(COMMENT_RE)
+            @col += comment.length
+          else
+            break
+          end
+        end
+      end
+
+      def scan_token
+        line = @line
+        col = @col
+
+        if @ss.peek(1) == '"'
+          value = scan_string_literal
+          return Token.new(kind: :STRING, value: value, line: line, column: col)
+        end
+
+        if op = @ss.scan(/==|!=/)
+          @col += op.length
+          return Token.new(kind: op == "==" ? :EQEQ : :NEQ, value: op, line: line, column: col)
+        end
+
+        char = @ss.peek(1)
+        if PUNCTUATION.key?(char)
+          @ss.getch
+          @col += 1
+          return Token.new(kind: PUNCTUATION[char], value: char, line: line, column: col)
+        end
+
+        if mc = @ss.scan(META_CLASS_RE)
+          @col += mc.length
+          return Token.new(kind: :META_CLASS, value: mc, line: line, column: col)
+        end
+
+        if irdi = @ss.scan(IRDI_RE)
+          @col += irdi.length
+          return Token.new(kind: :IRDI, value: irdi, line: line, column: col)
+        end
+
+        if local = @ss.scan(LOCAL_REF_RE)
+          @col += local.length
+          return Token.new(kind: :IDENT, value: local, line: line, column: col)
+        end
+
+        if date = @ss.scan(DATE_RE)
+          unless @ss.peek(1) =~ /[A-Za-z_]/
+            @col += date.length
+            return Token.new(kind: :DATE, value: date, line: line, column: col)
+          end
+          @ss.pos -= date.length
+        end
+
+        if num = @ss.scan(NUMBER_RE)
+          @col += num.length
+          return Token.new(kind: :NUMBER, value: num, line: line, column: col)
+        end
+
+        if ident = @ss.scan(IDENT_RE)
+          @col += ident.length
+          kind = KEYWORD_MAP[ident] || :IDENT
+          return Token.new(kind: kind, value: ident, line: line, column: col)
+        end
+
+        nil
+      end
+
+      def scan_string_literal
+        @ss.getch
+        buffer = +""
+        until @ss.eos?
+          ch = @ss.getch
+          case ch
+          when '"'
+            @col += buffer.length + 2
+            return buffer
+          when "\\"
+            esc = @ss.getch
+            raise LexError, "unterminated escape at line #{@line}" unless esc
+            buffer << decode_escape(esc)
+          when "\n"
+            raise LexError, "unterminated string at line #{@line}"
+          else
+            buffer << ch
+          end
+        end
+        raise LexError, "unterminated string at line #{@line}"
+      end
+
+      def decode_escape(esc)
+        case esc
+        when '"'  then '"'
+        when "\\" then "\\"
+        when "/"  then "/"
+        when "b"  then "\b"
+        when "f"  then "\f"
+        when "n"  then "\n"
+        when "r"  then "\r"
+        when "t"  then "\t"
+        when "u"
+          hex = @ss.scan(/[0-9a-fA-F]{4}/)
+          raise LexError, "invalid unicode escape at line #{@line}" unless hex
+          [hex.to_i(16)].pack("U")
+        else
+          raise LexError, "invalid escape \\#{esc} at line #{@line}"
+        end
+      end
+    end
+  end
+end

data/lib/opencdd/cddal/parser.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Cddal
+    class Parser
+      def self.parse(source)
+        new(Lexer.new(source).tokens).parse
+      end
+
+      def initialize(tokens)
+        @tokens = tokens
+      end
+
+      def parse
+        GeneratedParser.new(@tokens).parse
+      end
+    end
+  end
+end

data/lib/opencdd/cddal/resolver.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Opencdd
+  module Cddal
+    # Resolves a CDDAL module specifier to a canonical key and source
+    # text. The canonical key (absolute path or normalized URL) is
+    # used by the Builder to deduplicate imports — the same module
+    # imported twice loads only once.
+    #
+    # Resolution rules, in priority order:
+    #
+    #   1. URL (http://, https://, file://) — fetched via the
+    #      configured Fetcher. URL is the canonical key.
+    #   2. Absolute filesystem path — read directly. Expanded path
+    #      is the canonical key.
+    #   3. Relative path (starts with ./ or ../) — resolved against
+    #      the importing file's directory, falling back to base_path.
+    #   4. Bare name — searched in search_path entries, then
+    #      base_path. First match wins.
+    #
+    # When +strict:+ is false (default), URL fetch failures and
+    # missing paths emit a warning and return +[nil, nil]+ so the
+    # Builder can skip them. When +strict:+ is true, raises
+    # +Opencdd::Cddal::ImportError+ on any resolution failure.
+    class Resolver
+      DEFAULT_SEARCH_PATH = [Pathname.pwd].freeze
+
+      # When +strict:+ is false (default), URL fetch failures and
+      # missing paths emit a warning and return +[nil, nil]+ so the
+      # Builder can skip them. When +strict:+ is true, raises
+      # +Opencdd::Cddal::ImportError+ on any resolution failure.
+      # Pass +quiet: true+ to suppress the non-strict warning
+      # (used by tests that intentionally exercise unreachable URLs).
+      attr_reader :base_path, :search_path, :fetcher, :strict, :quiet
+
+      def initialize(base_path: nil, search_path: nil, fetcher: nil,
+                     strict: false, quiet: false)
+        @base_path   = base_path ? Pathname.new(base_path) : Pathname.pwd
+        @search_path = Array(search_path).map { |p| Pathname.new(p.to_s) }
+        @fetcher     = fetcher || Opencdd::Cddal::Fetcher::NetHttp.new
+        @strict      = strict
+        @quiet       = quiet
+      end
+
+      # Returns a tuple +[canonical_key, source_text]+. Returns
+      # +[nil, nil]+ in non-strict mode when resolution fails.
+      def resolve(specifier, importing_file: nil)
+        specifier = specifier.to_s
+        return resolve_url(specifier) if url?(specifier)
+
+        resolve_path(specifier, importing_file)
+      rescue Opencdd::Cddal::ImportError => e
+        raise if @strict
+        warn "CDDAL: #{e.message}" unless @quiet
+        [nil, nil]
+      end
+
+      private
+
+      def url?(s)
+        s.start_with?("http://", "https://", "file://")
+      end
+
+      def resolve_url(url)
+        text = @fetcher.fetch(url)
+        [url, text]
+      end
+
+      def resolve_path(spec, importing_file)
+        p = Pathname.new(spec)
+        candidates = path_candidates(p, importing_file)
+        candidates.uniq.each do |cand|
+          return [cand.expand_path.to_s, cand.read] if cand.exist? && cand.file?
+        end
+        raise Opencdd::Cddal::ImportError,
+              "cannot resolve import #{spec.inspect} " \
+              "(searched: #{candidates.map(&:to_s).join(', ')})"
+      end
+
+      def path_candidates(p, importing_file)
+        if p.absolute?
+          [p]
+        elsif spec_relative?(p)
+          base = importing_file ? Pathname.new(importing_file).dirname : @base_path
+          [base.join(p)]
+        else
+          # Bare name: walk search path, then base.
+          @search_path.map { |sp| sp.join(p) } + [@base_path.join(p)]
+        end
+      end
+
+      def spec_relative?(p)
+        s = p.to_s
+        s.start_with?("./", "../")
+      end
+    end
+  end
+end

data/lib/opencdd/cddal/serializer.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Cddal
+    class Serializer
+      # Matches exactly the values the CDDAL lexer can tokenize as a
+      # single non-string token: keywords, numbers, dates, IRDIs,
+      # local refs, and bare identifiers. Anything not matching must
+      # be emitted as a quoted string literal so the lexer can reparse
+      # it via scan_string_literal (which is UTF-8 aware via getch).
+      UNQUOTED_VALUE_RE = /\A(?:
+        true|false|null|
+        -?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|
+        \d{4}-\d{2}-\d{2}|
+        [0-9]+\/[0-9]+\/\/\/[A-Za-z0-9_]+(?:_[0-9]+)?\#[A-Za-z0-9_]+|
+        [A-Za-z_][A-Za-z0-9_]*\#[A-Za-z0-9_]+(?:\#\#[A-Za-z0-9_]+)?|
+        [A-Za-z_][A-Za-z0-9_]*
+      )\z/x.freeze
+
+      HEADER = <<~CDDAL
+        # Generated by Opencdd::Cddal::Serializer
+      CDDAL
+
+      attr_reader :database
+
+      def initialize(database)
+        @database = database
+      end
+
+      def to_cddal
+        lines = [HEADER.chomp]
+        emit_default_aliases(lines)
+        emit_value_lists(lines)
+        emit_value_terms(lines)
+        emit_units(lines)
+        emit_class_tree(lines)
+        emit_rooted_properties(lines)
+        emit_relations(lines)
+        emit_view_controls(lines)
+        lines.join("\n") + "\n"
+      end
+
+      private
+
+      def emit_default_aliases(lines)
+        lines << ""
+        Opencdd::PropertyIds::REGISTRY.each_value do |entry|
+          entry.aliases.each do |alias_name|
+            lines << "alias #{alias_name}: #{entry.id}"
+          end
+        end
+      end
+
+      def emit_value_lists(lines)
+        lists = @database.value_lists.sort_by { |vl| vl.code.to_s }
+        return if lists.empty?
+        lines << ""
+        lists.each do |vl|
+          lines.concat(emit_instance(vl))
+        end
+      end
+
+      def emit_value_terms(lines)
+        terms = @database.value_terms.sort_by { |vt| vt.code.to_s }
+        return if terms.empty?
+        lines << ""
+        terms.each do |vt|
+          lines.concat(emit_instance(vt))
+        end
+      end
+
+      def emit_units(lines)
+        units = @database.units.sort_by { |u| u.code.to_s }
+        return if units.empty?
+        lines << ""
+        units.each do |u|
+          lines.concat(emit_instance(u))
+        end
+      end
+
+      def emit_class_tree(lines)
+        roots = @database.root_classes.sort_by { |k| k.code.to_s }
+        return if roots.empty?
+        lines << ""
+        roots.each do |root|
+          walk_class(root, lines, 0)
+        end
+      end
+
+      def walk_class(klass, lines, depth)
+        lines.concat(emit_instance(klass))
+        sorted_children = klass.children.sort_by { |c| c.code.to_s }
+        sorted_children.each { |c| walk_class(c, lines, depth + 1) }
+      end
+
+      def emit_rooted_properties(lines)
+        emitted = Set.new
+        @database.classes.sort_by { |k| k.code.to_s }.each do |klass|
+          props = @database.properties_of(klass).sort_by { |p| p.code.to_s }
+          props.each do |prop|
+            next if emitted.include?(prop.irdi)
+            emitted << prop.irdi
+            lines.concat(emit_instance(prop))
+          end
+        end
+        unrooted = @database.properties.reject { |p| emitted.include?(p.irdi) }.sort_by { |p| p.code.to_s }
+        unrooted.each { |p| lines.concat(emit_instance(p)) }
+      end
+
+      def emit_relations(lines)
+        rels = @database.relations.sort_by { |r| r.code.to_s }
+        return if rels.empty?
+        lines << ""
+        rels.each { |r| lines.concat(emit_instance(r)) }
+      end
+
+      def emit_view_controls(lines)
+        vcs = @database.view_controls.sort_by { |v| v.code.to_s }
+        return if vcs.empty?
+        lines << ""
+        vcs.each { |v| lines.concat(emit_instance(v)) }
+      end
+
+      def emit_instance(entity)
+        name = symbol_name_for(entity)
+        meta = meta_class_for(entity)
+        out = []
+        out << ""
+        out << "instance #{name} < #{meta} {"
+        emit_properties(entity).each { |line| out << "  #{line}" }
+        out << "}"
+        out
+      end
+
+      def emit_properties(entity)
+        out = []
+        entity.properties.each do |key, value|
+          next if value.nil? || value.to_s.empty?
+          name = key.to_s
+          base = name.sub(/\.\w+\z/, "")
+          lang_match = name.match(/\.(?<lang>[a-zA-Z]{2,3}(?:-[A-Za-z0-9]+)?)\z/)
+          lang = lang_match && lang_match[:lang]
+          alias_name = alias_for_property_id(base)
+          display_key = lang ? "#{alias_name}.#{lang}" : alias_name
+          rhs = format_value(value)
+          out << "#{display_key}: #{rhs}"
+        end
+        out
+      end
+
+      def alias_for_property_id(property_id)
+        name = property_id.to_s
+        base = name.sub(/\.\w+\z/, "")
+        Opencdd::PropertyIds.entry(base)&.aliases&.first || name
+      end
+
+      def language_suffix(property_id)
+        m = property_id.to_s.match(/\.(?<lang>[a-zA-Z]{2,3}(-[A-Za-z0-9]+)?)\z/)
+        m && m[:lang]
+      end
+
+      def format_value(value)
+        s = value.to_s
+        return quote_string(s) if string_literal?(s)
+        return format_set(s) if set_like?(s)
+        s
+      end
+
+      def string_literal?(s)
+        !s.match?(UNQUOTED_VALUE_RE)
+      end
+
+      def quote_string(s)
+        escaped = s
+          .gsub("\\") { "\\\\" }
+          .gsub('"')  { "\\\"" }
+          .gsub("\n") { "\\n" }
+          .gsub("\r") { "\\r" }
+          .gsub("\t") { "\\t" }
+          .gsub("\b") { "\\b" }
+          .gsub("\f") { "\\f" }
+        "\"#{escaped}\""
+      end
+
+      def set_like?(s)
+        s.start_with?("(") && s.end_with?(")")
+      end
+
+      def format_set(s)
+        elements = Opencdd::StructuredValues.unwrap_and_split(s)
+        "{ #{elements.join(', ')} }"
+      end
+
+      def symbol_name_for(entity)
+        code = entity.code&.to_s
+        return code if code && !code.empty?
+        preferred = entity.preferred_name&.to_s
+        return "instance" unless preferred
+        safe = preferred.gsub(/[^A-Za-z0-9_]/, "_").gsub(/\A(\d)/, "_\\1")
+        safe.empty? ? "instance" : safe
+      end
+
+      def meta_class_for(entity)
+        type = entity.type
+        code = Opencdd::MetaClasses.meta_class_for_type(type) if type
+        code || entity.meta_class_irdi&.code || Opencdd::MetaClasses::MDC_C002
+      end
+    end
+  end
+end

data/lib/opencdd/cddal/value_serializer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Cddal
+    # Pure-function value serializer: converts AST value nodes into
+    # the CDDAL/Parcel wire string form. Extracted from Builder
+    # (TODO.impl/28) so the builder's property-assembly pipeline
+    # is a consumer of this module rather than owning the logic.
+    #
+    # Stateless — safe to call from any context. The Builder mixes
+    # this in for backward compatibility.
+    module ValueSerializer
+      module_function
+
+      def serialize(value, property_id = nil)
+        case value
+        when AST::Literal
+          value.raw
+        when AST::IdentifierRef
+          value.to_s
+        when AST::Set
+          elements = value.elements.map { |e| serialize_set_element(e) }
+          "{#{elements.join(',')}}"
+        when AST::Tuple
+          elements = value.elements.map { |e| serialize_tuple_element(e) }
+          "(#{elements.join(',')})"
+        when AST::ClassReference
+          argument = case value.argument
+                     when AST::IdentifierRef then value.argument.name
+                     else value.argument.to_s
+                     end
+          "#{value.type_name}(#{argument})"
+        when AST::Condition
+          rhs = value.right.to_cddal
+          "#{value.left} #{value.operator} #{rhs}"
+        else
+          value.to_s
+        end
+      end
+
+      def serialize_set_element(element)
+        case element
+        when AST::IdentifierRef then element.to_s
+        when AST::Literal       then element.raw
+        when AST::Tuple         then serialize(element, nil)
+        when AST::Set           then serialize(element, nil)
+        else element.to_s
+        end
+      end
+
+      def serialize_tuple_element(element)
+        case element
+        when AST::IdentifierRef then element.to_s
+        when AST::Literal       then element.raw
+        else element.to_s
+        end
+      end
+    end
+  end
+end

data/lib/opencdd/cddal.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Cddal
+    autoload :Lexer,           "opencdd/cddal/lexer"
+    autoload :AST,             "opencdd/cddal/ast"
+    autoload :Parser,          "opencdd/cddal/parser"
+    autoload :GeneratedParser, "opencdd/cddal/generated_parser"
+    autoload :Builder,         "opencdd/cddal/builder"
+    autoload :Serializer,      "opencdd/cddal/serializer"
+    autoload :Resolver,        "opencdd/cddal/resolver"
+    autoload :Fetcher,         "opencdd/cddal/fetcher"
+    autoload :ValueSerializer, "opencdd/cddal/value_serializer"
+
+    class Error < StandardError; end
+    class LexError < Error; end
+    class ParseError < Error; end
+    class ResolutionError < Error; end
+    class ImportError < Error; end
+
+    @default_resolver = nil
+
+    module_function
+
+    def default_resolver
+      @default_resolver ||= Resolver.new
+    end
+
+    # Parse +source+ (a CDDAL string) into a +Opencdd::Database+.
+    #
+    # Options:
+    #   database: - existing Database to merge into (default: new)
+    #   resolver: - Opencdd::Cddal::Resolver for module imports
+    #               (default: a fresh Resolver with default fetcher)
+    #   fetcher:  - shortcut; if supplied, wraps in a new Resolver
+    #   source_file: - path/URL the source came from. Used for
+    #               diagnostics and for resolving relative imports.
+    def parse(source, database: nil, resolver: nil, fetcher: nil,
+              source_file: nil)
+      tokens = Lexer.new(source.to_s).tokens
+      declarations = Parser.new(tokens).parse
+      effective_resolver =
+        resolver || (fetcher ? Resolver.new(fetcher: fetcher) : default_resolver)
+      Builder.new(database, resolver: effective_resolver, source_file: source_file)
+        .build(declarations)
+    end
+
+    def parse_file(path, database: nil, resolver: nil, fetcher: nil)
+      source = File.read(path)
+      parse(source, database: database,
+            resolver: resolver, fetcher: fetcher, source_file: path)
+    end
+
+    def serialize(database)
+      Serializer.new(database).to_cddal
+    end
+
+    def serialize_to_file(database, path)
+      File.write(path, serialize(database))
+      self
+    end
+  end
+end