openusd 1.0.0

data/lib/openusd/format/usda/lexer.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module OpenUSD
+  module Format
+    # USDA text format support.
+    module Usda
+      # Converts USDA text into location-aware tokens.
+      class Lexer
+        # One lexeme and its source location.
+        Token = Data.define(:type, :value, :line, :column, :raw)
+        # Single-character grammar symbols.
+        SYMBOLS = "(){}[]=,:."
+        # Numeric literal pattern shared by ordinary and fast-path scans.
+        NUMERIC_LITERAL = /[+-]?(?:(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?|inf|nan)/
+        # Supported string escape translations.
+        ESCAPES = {
+          "n" => "\n", "r" => "\r", "t" => "\t", "b" => "\b",
+          "f" => "\f", "\\" => "\\", "\"" => "\""
+        }.freeze
+
+        attr_reader :file
+
+        def initialize(source, file: nil)
+          @scanner = StringScanner.new(String(source).delete_prefix("\uFEFF"))
+          @file = file
+          @line = 1
+          @column = 1
+        end
+
+        # Iterate through all tokens, including the final EOF token.
+        def each_token
+          return enum_for(__method__) unless block_given?
+
+          loop do
+            token = next_token
+            yield token
+            break if token.type == :eof
+          end
+        end
+
+        # @return [Token] next token from the source
+        def next_token
+          skip_ignored
+          return token(:eof, nil, "") if @scanner.eos?
+
+          return scan_magic if @scanner.match?(/#usda\b/)
+          return scan_quoted("\"\"\"", :string) if @scanner.peek(3) == "\"\"\""
+          return scan_quoted("\"", :string) if @scanner.peek(1) == "\""
+          return scan_quoted("@@@", :asset) if @scanner.peek(3) == "@@@"
+          return scan_quoted("@", :asset) if @scanner.peek(1) == "@"
+          return scan_path if @scanner.peek(1) == "<"
+
+          scan_bare_token
+        end
+
+        # Scan a flat or tuple-grouped numeric array without allocating tokens.
+        # @api private
+        # @return [Array, nil] values, or nil when the input needs normal lexing
+        def scan_numeric_array(component_count: nil)
+          rest = @scanner.rest
+          closing = rest.index("]")
+          return unless closing
+
+          body = rest[0...closing]
+          literals = body.scan(NUMERIC_LITERAL)
+          residue = body.gsub(NUMERIC_LITERAL, "").gsub(/[\s,()]/, "")
+          return unless residue.empty?
+          return if component_count && (literals.length % component_count).nonzero?
+
+          raw = rest[0..closing]
+          @scanner.pos += raw.bytesize
+          advance(raw)
+          values = literals.map { |literal| numeric_value(literal) }
+          component_count ? values.each_slice(component_count).to_a : values
+        end
+
+        private
+
+        def scan_bare_token
+          return scan_special_number if @scanner.match?(/[+-]?(?:inf|nan)\b/)
+          return scan_number if @scanner.match?(/[+-]?(?:\d|\.\d)/)
+          return scan_identifier if @scanner.match?(/[\p{L}_]/u)
+          return scan_symbol if SYMBOLS.include?(@scanner.peek(1))
+
+          lexical_error!("unexpected character #{@scanner.peek(1).inspect}")
+        end
+
+        def skip_ignored
+          loop do
+            skipped = scan_ignored_pattern?(/[ \t\r\n]+/)
+            skipped ||= scan_line_comment?("#") unless @scanner.match?(/#usda\b/)
+            skipped ||= scan_line_comment?("//")
+            skipped ||= scan_block_comment?
+            break unless skipped
+          end
+        end
+
+        def scan_ignored_pattern?(pattern)
+          raw = @scanner.scan(pattern)
+          return false unless raw
+
+          advance(raw)
+          true
+        end
+
+        def scan_line_comment?(prefix)
+          return false unless @scanner.peek(prefix.length) == prefix
+
+          raw = @scanner.scan(/.*(?:\n|\z)/)
+          advance(raw)
+          true
+        end
+
+        def scan_block_comment?
+          return false unless @scanner.peek(2) == "/*"
+
+          start_line = @line
+          start_column = @column
+          raw = @scanner.scan(%r{/\*.*?\*/}m)
+          lexical_error!("unterminated block comment", start_line, start_column) unless raw
+          advance(raw)
+          true
+        end
+
+        def scan_magic
+          line = @line
+          column = @column
+          raw = @scanner.scan(/#usda[ \t]+([0-9]+(?:\.[0-9]+)?)/)
+          lexical_error!("invalid USDA header", line, column) unless raw
+          version = @scanner[1]
+          advance(raw)
+          Token.new(:magic, version, line, column, raw)
+        end
+
+        def scan_quoted(delimiter, type)
+          line = @line
+          column = @column
+          raw = consume(delimiter)
+          value = +""
+
+          until @scanner.eos?
+            if @scanner.peek(delimiter.length) == delimiter
+              raw << consume(delimiter)
+              return Token.new(type, value.freeze, line, column, raw.freeze)
+            end
+
+            character = consume_character
+            raw << character
+            if character == "\\" && type == :string
+              escaped = consume_character
+              raw << escaped
+              value << ESCAPES.fetch(escaped, escaped)
+            else
+              value << character
+            end
+          end
+
+          lexical_error!("unterminated #{type}", line, column)
+        end
+
+        def scan_path
+          line = @line
+          column = @column
+          raw = consume("<")
+          value = +""
+
+          until @scanner.eos?
+            character = consume_character
+            raw << character
+            return Token.new(:path, value.freeze, line, column, raw.freeze) if character == ">"
+
+            value << character
+          end
+
+          lexical_error!("unterminated path", line, column)
+        end
+
+        def scan_number
+          line = @line
+          column = @column
+          raw = @scanner.scan(/[+-]?(?:(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?)/)
+          value = raw.match?(/[.eE]/) ? Float(raw) : Integer(raw, 10)
+          advance(raw)
+          Token.new(:number, value, line, column, raw.freeze)
+        end
+
+        def scan_special_number
+          line = @line
+          column = @column
+          raw = @scanner.scan(/[+-]?(?:inf|nan)\b/)
+          value = if raw.include?("nan")
+                    Float::NAN
+                  elsif raw.start_with?("-")
+                    -Float::INFINITY
+                  else
+                    Float::INFINITY
+                  end
+          advance(raw)
+          Token.new(:number, value, line, column, raw.freeze)
+        end
+
+        def numeric_value(raw)
+          return Float::NAN if raw.include?("nan")
+          return raw.start_with?("-") ? -Float::INFINITY : Float::INFINITY if raw.include?("inf")
+
+          raw.match?(/[.eE]/) ? Float(raw) : Integer(raw, 10)
+        end
+
+        def scan_identifier
+          line = @line
+          column = @column
+          raw = @scanner.scan(/[\p{L}_][\p{L}\p{N}_:]*(?:\[\])?/u)
+          advance(raw)
+          Token.new(:identifier, raw.freeze, line, column, raw.freeze)
+        end
+
+        def scan_symbol
+          line = @line
+          column = @column
+          raw = consume_character
+          Token.new(:symbol, raw, line, column, raw)
+        end
+
+        def consume(expected)
+          raw = @scanner.scan(Regexp.new(Regexp.escape(expected)))
+          advance(raw)
+          raw
+        end
+
+        def consume_character
+          raw = @scanner.getch
+          advance(raw)
+          raw
+        end
+
+        def advance(raw)
+          lines = raw.count("\n")
+          if lines.zero?
+            @column += raw.length
+          else
+            @line += lines
+            @column = raw.length - raw.rindex("\n")
+          end
+        end
+
+        def token(type, value, raw)
+          Token.new(type, value, @line, @column, raw)
+        end
+
+        def lexical_error!(message, line = @line, column = @column)
+          raise ParseError.new(message, file: file, line: line, column: column)
+        end
+      end
+    end
+  end
+end

data/lib/openusd/format/usda/parser.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+module OpenUSD
+  module Format
+    module Usda
+      # Recursive-descent parser for USDA layers.
+      class Parser
+        # Authored prim specifier keywords.
+        SPECIFIERS = %w[def over class].freeze
+        # Recognized list-edit operation keywords.
+        LIST_OPERATIONS = %w[prepend append delete reorder add explicit].freeze
+        # Recognized property qualifiers.
+        QUALIFIERS = %w[custom uniform varying].freeze
+
+        class << self
+          # Parse a USDA string.
+          # @return [Layer]
+          def parse(source, file: nil)
+            new(source, file: file).parse
+          end
+        end
+
+        attr_reader :file
+
+        def initialize(source, file: nil)
+          @lexer = Lexer.new(source, file: file)
+          @file = file
+          advance
+        end
+
+        # Parse the configured source.
+        # @return [Layer]
+        def parse
+          header = expect(:magic, description: "USDA header")
+          error!("unsupported USDA version #{header.value}") unless header.value == "1.0"
+
+          layer = Layer.new(file)
+          layer.metadata.merge!(parse_metadata_block(layer_comment: true)) if symbol?("(")
+          layer.add_root_prim(parse_prim) until current.type == :eof
+          layer
+        rescue OpenUSD::PathError, OpenUSD::TypeError => e
+          error!(e.message)
+        end
+
+        private
+
+        attr_reader :current
+
+        def parse_prim
+          specifier = expect_identifier
+          error!("expected prim specifier") unless SPECIFIERS.include?(specifier)
+
+          type_name = expect_identifier unless current.type == :string
+          name = expect(:string, description: "prim name").value
+          metadata = symbol?("(") ? parse_metadata_block : {}
+          references, reference_list_op, references_authored = ReferenceMetadata.extract(metadata)
+          prim = PrimSpec.new(
+            name,
+            type_name: type_name,
+            specifier: specifier,
+            metadata: metadata,
+            references: references_authored ? references : PrimSpec::REFERENCES_UNAUTHORED,
+            reference_list_op: reference_list_op
+          )
+          parse_prim_body(prim)
+          prim
+        end
+
+        def parse_prim_body(prim)
+          expect_symbol("{")
+          until accept_symbol?("}")
+            if specifier?
+              prim.add_child(parse_prim)
+            elsif identifier?("variantSet")
+              parse_variant_set(prim)
+            else
+              PropertyMerger.merge(prim, parse_property)
+            end
+          end
+        end
+
+        def parse_variant_set(prim)
+          advance
+          name = expect(:string, description: "variant set name").value
+          expect_symbol("=")
+          expect_symbol("{")
+          choices = {}
+          until accept_symbol?("}")
+            choice = expect(:string, description: "variant name").value
+            holder = PrimSpec.new("_variant")
+            parse_prim_body(holder)
+            choices[choice] = Variant.new(holder.properties.freeze, holder.children.freeze)
+          end
+          prim.variant_sets[name] = choices
+        end
+
+        def parse_property
+          qualifiers = []
+          qualifiers << consume_identifier while current.type == :identifier && QUALIFIERS.include?(current.value)
+
+          return parse_relationship(qualifiers) if identifier?("rel")
+
+          type_name = expect_identifier
+          name = expect_identifier
+          suffix = parse_property_suffix
+          value = parse_assigned_value(type_name)
+          metadata = symbol?("(") ? parse_metadata_block : {}
+          build_attribute(name, type_name, suffix, value, qualifiers, metadata)
+        end
+
+        def parse_relationship(qualifiers)
+          advance
+          name = expect_identifier
+          value = accept_symbol?("=") ? parse_value : RelationshipSpec::UNAUTHORED
+          metadata = symbol?("(") ? parse_metadata_block : {}
+          targets = value.equal?(RelationshipSpec::UNAUTHORED) ? value : unwrap_list(value).compact
+          RelationshipSpec.new(name, targets: targets, custom: qualifiers.include?("custom"), metadata: metadata)
+        end
+
+        def parse_property_suffix
+          return unless accept_symbol?(".")
+
+          expect_identifier
+        end
+
+        def parse_assigned_value(type_name)
+          return AttributeSpec::UNAUTHORED unless accept_symbol?("=")
+
+          parse_value(type_name)
+        end
+
+        def build_attribute(name, type_name, suffix, value, qualifiers, metadata)
+          attribute = AttributeSpec.new(
+            name, type_name,
+            variability: qualifiers.include?("uniform") ? :uniform : :varying,
+            custom: qualifiers.include?("custom"),
+            metadata: metadata
+          )
+          apply_attribute_value(attribute, suffix, value)
+          attribute
+        end
+
+        def apply_attribute_value(attribute, suffix, value)
+          case suffix
+          when "timeSamples"
+            attribute.time_samples = value
+          when "connect"
+            attribute.connections = unwrap_list(value)
+          when nil
+            attribute.set(value) unless value.equal?(AttributeSpec::UNAUTHORED)
+          else
+            attribute.metadata[suffix] = value
+          end
+        end
+
+        def parse_metadata_block(layer_comment: false)
+          expect_symbol("(")
+          metadata = {}
+          until accept_symbol?(")")
+            if layer_comment && current.type == :string
+              error!("duplicate layer comment") if metadata.key?("comment")
+
+              metadata["comment"] = consume.value
+            else
+              metadata.merge!(parse_metadata_entry)
+            end
+          end
+          metadata
+        end
+
+        def parse_metadata_entry
+          operation = consume_identifier if current.type == :identifier && LIST_OPERATIONS.include?(current.value)
+          first = expect_identifier
+          type_name, key = metadata_type_and_key(first)
+          expect_symbol("=")
+          value = parse_value(type_name)
+          value = ListOp.new(operation, value) if operation
+          { key => value }
+        end
+
+        def metadata_type_and_key(first)
+          return [nil, first] unless current.type == :identifier
+
+          [first, consume_identifier]
+        end
+
+        def parse_value(expected_type = nil)
+          case current.type
+          when :string, :number then consume.value
+          when :asset then parse_asset
+          when :path then Path.parse(consume.value)
+          when :identifier then parse_identifier_value
+          when :symbol then parse_compound_value(expected_type)
+          else error!("expected a value")
+          end
+        end
+
+        def parse_asset
+          asset = AssetPath.new(consume.value)
+          return asset unless current.type == :path
+
+          Reference.new(asset, Path.parse(consume.value))
+        end
+
+        def parse_identifier_value
+          value = consume.value
+          return true if value == "true"
+          return false if value == "false"
+          return nil if value == "None"
+          return Float::INFINITY if value == "inf"
+          return -Float::INFINITY if value == "-inf"
+          return Float::NAN if value == "nan"
+
+          Token.new(value)
+        end
+
+        def parse_compound_value(expected_type)
+          return parse_sequence("(", ")", expected_type) if symbol?("(")
+          return parse_array(expected_type) if symbol?("[")
+          return parse_dictionary if symbol?("{")
+
+          error!("expected a value")
+        end
+
+        def parse_array(expected_type)
+          base_type = Types.base_type(expected_type.to_s)
+          component_count = Types::VECTOR_TYPES[base_type]
+          numeric = component_count || Types::FLOAT_TYPES.include?(base_type) ||
+                    Types::INTEGER_RANGES.key?(base_type)
+          return parse_sequence("[", "]", base_type) unless numeric
+
+          values = @lexer.scan_numeric_array(component_count: component_count)
+          return parse_sequence("[", "]", base_type) unless values
+
+          advance
+          values
+        end
+
+        def parse_sequence(opening, closing, expected_type)
+          expect_symbol(opening)
+          values = []
+          until accept_symbol?(closing)
+            values << parse_value(expected_type)
+            accept_symbol?(",")
+          end
+          values
+        end
+
+        def parse_dictionary
+          expect_symbol("{")
+          values = {}
+          until accept_symbol?("}")
+            key, type_name, separator = parse_dictionary_key
+            expect_symbol(separator)
+            values[key] = parse_value(type_name)
+            accept_symbol?(",")
+          end
+          values
+        end
+
+        def parse_dictionary_key
+          return [consume.value, nil, ":"] if current.type == :number
+
+          first = consume
+          error!("expected dictionary key") unless %i[string identifier].include?(first.type)
+          return [first.value, nil, current.value] if symbol?(":") || symbol?("=")
+
+          [expect_identifier, first.value, "="]
+        end
+
+        def unwrap_list(value)
+          value = value.value if value.is_a?(ListOp)
+          value.is_a?(Array) ? value : [value]
+        end
+
+        def specifier? = current.type == :identifier && SPECIFIERS.include?(current.value)
+
+        def identifier?(value) = current.type == :identifier && current.value == value
+
+        def symbol?(value) = current.type == :symbol && current.value == value
+
+        def accept_symbol?(value)
+          return false unless symbol?(value)
+
+          advance
+          true
+        end
+
+        def expect_symbol(value)
+          expect(:symbol, value: value, description: value)
+        end
+
+        def expect_identifier
+          expect(:identifier, description: "identifier").value
+        end
+
+        def consume_identifier
+          expect_identifier
+        end
+
+        def expect(type, value: nil, description: type)
+          token = current
+          matches = token.type == type && (value.nil? || token.value == value)
+          error!("expected #{description}, got #{token.raw.inspect}") unless matches
+          advance
+          token
+        end
+
+        def consume
+          token = current
+          advance
+          token
+        end
+
+        def advance
+          @current = @lexer.next_token
+        end
+
+        def error!(message)
+          raise ParseError.new(message, file: file, line: current.line, column: current.column)
+        end
+      end
+
+      # File reader adapter for the format registry.
+      module Reader
+        module_function
+
+        # Read a USDA layer from a filesystem path.
+        # @return [Layer]
+        def read(path)
+          Parser.parse(File.binread(path).force_encoding(Encoding::UTF_8), file: path)
+        rescue Errno::ENOENT
+          raise CompositionError, "layer not found: #{path}"
+        end
+      end
+    end
+  end
+end

data/lib/openusd/format/usda/property_merger.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module OpenUSD
+  module Format
+    module Usda
+      # Combines fields from repeated USDA declarations into one property spec.
+      module PropertyMerger
+        module_function
+
+        # Merge a parsed property into its owning prim.
+        # @return [AttributeSpec, RelationshipSpec]
+        def merge(prim, property)
+          existing = prim.property_named(property.name)
+          return prim.add_property(property) unless existing
+
+          unless existing.instance_of?(property.class)
+            raise OpenUSD::TypeError, "property #{property.name} is declared with conflicting kinds"
+          end
+
+          existing.is_a?(AttributeSpec) ? merge_attribute(existing, property) : merge_relationship(existing, property)
+          existing
+        end
+
+        def merge_attribute(existing, property)
+          unless existing.type_name == property.type_name
+            raise OpenUSD::TypeError, "attribute #{property.name} is declared with conflicting types"
+          end
+
+          existing.custom ||= property.custom
+          existing.variability = :uniform if property.variability == :uniform
+          existing.metadata.merge!(property.metadata)
+          existing.set(property.default) if property.default_authored?
+          existing.time_samples = property.time_samples if property.time_samples_authored?
+          existing.connections = property.connections if property.connections_authored?
+        end
+        private_class_method :merge_attribute
+
+        def merge_relationship(existing, property)
+          existing.custom ||= property.custom
+          existing.metadata.merge!(property.metadata)
+          existing.targets = property.targets if property.targets_authored?
+        end
+        private_class_method :merge_relationship
+      end
+    end
+  end
+end

data/lib/openusd/format/usda/reference_metadata.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module OpenUSD
+  module Format
+    module Usda
+      # Converts a USDA references metadata field to model references.
+      module ReferenceMetadata
+        module_function
+
+        # Remove and normalize references metadata.
+        # @return [Array(Array<Reference>, Symbol, Boolean)]
+        def extract(metadata)
+          return [[], nil, false] unless metadata.key?("references")
+
+          value = metadata.delete("references")
+          operation = value.operation if value.is_a?(ListOp)
+          value = value.value if value.is_a?(ListOp)
+          references = Array(unwrap_list(value)).map { |reference| normalize(reference) }
+          [references, operation, true]
+        end
+
+        def normalize(reference)
+          return Reference.internal(reference) if reference.is_a?(Path)
+          return reference if reference.is_a?(Reference)
+
+          Reference.new(reference)
+        end
+        private_class_method :normalize
+
+        def unwrap_list(value)
+          value.is_a?(Array) ? value : [value]
+        end
+        private_class_method :unwrap_list
+      end
+    end
+  end
+end