fixture_fox 0.1.1

data/lib/fixture_fox/hash_parser.rb

@@ -0,0 +1,111 @@
+class HashParser
+  def self.parse(s)
+    self.new(s).send(:parse)
+  end
+
+private
+  def initialize(s)
+    @s, @i = s, 0
+  end
+
+  def parse
+    @s[@i] == "{" or raise "Missing initial '{' in hash literal: #{@s.inspect}"
+    @i += 1
+    pairs = []
+    while @i < @s.size && @s[@i] !~ /[,\}]/
+      pairs << extract_key_value
+      extract_element_separator
+    end
+    @s[@i] == "}" or raise "Missing terminating '}' in hash literal: #{@s.inspect}"
+    Hash[*pairs.flatten]
+  end
+
+  def extract_element_separator
+    while @i < @s.size && @s[@i] =~ /\s/
+      @i += 1
+    end
+    if @i < @s.size && @s[@i] == ","
+      @i += 1
+      while @i < @s.size && @s[@i] =~ /\s/
+        @i += 1
+      end
+    end
+  end
+
+  def extract_key_value
+    key = extract_key
+    extract_separator
+    value = extract_value
+    [key.to_sym, value]
+  end
+  
+  def extract_key
+    key = ""
+    while @i < @s.size && @s[@i] =~ /\w/
+      key += @s[@i]
+      @i += 1
+    end
+    key
+  end
+
+  def extract_separator
+    while @i < @s.size && @s[@i] =~ /[\s:]/
+      @i += 1
+    end
+  end
+
+  def extract_value
+    if @s[@i] =~ /['"]/
+      extract_quoted_value
+    else
+      extract_unquoted_value
+    end
+  end
+
+  def extract_quoted_value
+    quot = @s[@i]
+    @i += 1
+
+    litt = ""
+    while @i < @s.size && @s[@i] != quot
+      if @s[@i] == "\\"
+        @i += 1
+        @i < @s.size or raise "Syntax error in hash literal: #{@s.inspect}"
+        case @s[@i]
+          when "\\"; litt += "\\"
+          when quot; litt += quot
+        else
+          litt += "\\" + @s[@i]
+        end
+      else
+        litt += @s[@i]
+      end
+      @i += 1
+    end
+
+    @i < @s.size && @s[@i] == quot or raise "Unterminated hash literal: #{@s.inspect}"
+    @i += 1
+    return litt
+  end
+
+  def extract_unquoted_value
+    j = @i + 1
+    while j < @s.size && @s[j] !~ /[,\}]/
+      j += 1
+    end
+    litt = @s[@i...j].strip
+    @i = j
+    value =
+        case litt
+          when "nil"; nil
+          when "true"; true
+          when "false"; false
+          when /^\d+$/; litt.to_i
+        else
+          litt.strip
+        end
+    value 
+  end
+end
+
+

data/lib/fixture_fox/idr.rb

@@ -0,0 +1,62 @@
+
+require "tempfile"
+
+module FixtureFox
+  class Idr
+    # Qualified names of controlled tables
+    def tables() @tables_hash.keys end
+
+    # List of materialized views that depends on the tables. Assigned by the analyzer
+    # FIXME: Is this in use?
+    attr_accessor :materialized_views
+    
+    # Data as a hash from schema to table to id to record to field to value.
+    # Ie. { "schema" => { "table" => { 1 => { id: 1, name: "Alice" } } } }
+    attr_reader :data
+
+    # Map from qualified table name to last used ID. FIXME: Unused?
+    attr_reader :ids
+
+    def initialize
+      @tables_hash = {}
+      @data = {}
+    end
+
+    # Number of records
+    def count()
+      count = 0
+      @data.each { |_, tables| tables.each { |_, records| count += records.size } }
+      count
+    end
+
+    # Add a field to the data representation. If field and value are nil, an
+    # empty record will be created. If id is also nil, an empty table is
+    # created
+    #
+    def put(schema, table, id = nil, field = nil, value = nil)
+#     puts "Idr.put(#{schema}, #{table}, #{id.inspect}, #{field.inspect}, #{value.inspect})"
+      !id.nil? || field.nil? or raise ArgumentError
+      raise if table.to_s =~ /[A-Z]/
+      uid = "#{schema}.#{table}"
+      @tables_hash[uid] = true
+      return if id.nil?
+      tuple = ((@data[schema] ||= {})[table] ||= {})[id] ||= {id: id}
+      tuple[field.to_sym] = value if field
+    end
+
+    alias_method :to_h, :data
+
+    def dump
+      data.sort_by(&:first).each { |schema, tables|
+        puts schema
+        tables.each { |table, records| 
+          puts "  #{table}"
+          records.each { |id, fields|
+            puts "    #{fields.inspect}"
+          }
+        }
+      }
+    end
+  end
+end
+

data/lib/fixture_fox/line.rb

@@ -0,0 +1,217 @@
+module FixtureFox
+  class TokenizedLine
+    attr_reader :file
+    attr_reader :line
+    attr_reader :lineno
+    attr_reader :initial_indent
+    attr_reader :indent
+
+    def initialize(file, lineno, initial_indent, indent, line)
+      @file, @line, @lineno, @initial_indent, @indent = file, line.dup, lineno, initial_indent, indent
+      @pos = 1 + @indent # Used to keep track of position for the make_* methods
+    end
+
+    def error(msg)
+      raise ParseError.new(@file, @lineno, @pos + initial_indent, msg)
+    end
+
+    def to_s(long: false)
+      if long
+        "#{file}:#{lineno} "
+      else
+        "line #{lineno} "
+      end
+    end
+
+  protected
+    def make_token(litt)
+      Token.new(file, lineno, initial_indent, @pos, litt)
+    end
+
+    def make_ident(litt)
+      Ident.new(file, lineno, initial_indent, @pos, litt)
+    end
+
+    def make_empty(litt)
+      Empty.new(file, lineno, initial_indent, @pos, litt)
+    end
+
+    def make_value(litt)
+      Value.new(file, lineno, initial_indent, @pos, litt)
+    end
+
+    def make_directive(litt)
+      Directive.new(file, lineno, initial_indent, @pos, litt)
+    end
+
+    def make_anchor(litt)
+      AnchorToken.new(@file, @lineno, initial_indent, @pos, litt)
+    end
+
+    def make_reference(litt)
+      Reference.new(@file, @lineno, initial_indent, @pos, litt)
+    end
+  end
+
+  class DirectiveLine < TokenizedLine
+    attr_reader :directive # Directive token
+    attr_reader :argument # Value token
+
+    def initialize(file, lineno, initial_indent, indent, line)
+      super
+      case line
+        when /^(@schema)(\s+)(\w+)\s*(?:#.*)?$/
+          @directive = make_directive($1)
+          @pos += $1.size + $2.size
+          @argument = make_value($3)
+        when /^(@include)(\s+)(\S+)\s*(?:#.*)?$/
+          @directive = make_directive($1)
+          @pos += $1.size + $2.size
+          @argument = make_value($3)
+        when /^(@anchors)(\s+)(\S+)\s*(?:#.*)?$/
+          @directive = make_directive($1)
+          @pos += $1.size + $2.size
+          @argument = make_value($3)
+        else
+          error("Illegal directive")
+      end
+    end
+
+    def to_s(long: false)
+      super +
+          if long
+            "directive:#{directive.pos} #{directive.litt}, argument:#{argument.pos} #{argument.litt}"
+          else
+            "#{directive.litt} #{argument.litt}"
+          end
+    end
+  end
+
+  class Line < TokenizedLine
+    attr_reader :dash 
+    attr_reader :ident
+    attr_reader :empty # Empty token
+    attr_reader :value 
+    attr_reader :reference 
+    attr_reader :anchor
+
+    attr_reader :directive # SchemaDirective token
+
+    def initialize(file, lineno, initial_indent, indent, line)
+      super
+
+      # Remove comments not preceded by a quote (' or ")
+      if line =~ /^([^'"#]*?)\s*#.*/
+        line = $1
+      end
+
+      # Parse root element
+      if indent == 0 && line[0] != "-"
+        # Table
+        case line
+          when /^(\w+(?:\.\w+)?)$/
+            @ident = make_ident($1)
+            return
+          when /^(\w+(?:\.\w+)?)(\s*:\s*)(\[\])$/
+            @ident = make_ident($1)
+            @pos += $1.size + $2.size
+            @empty = make_empty($3)
+            return
+        end
+
+        # Check for missing value in root record
+        if line =~ /^(\w+\s*:)$/
+          error "Table name expected"
+        end
+      end
+
+      # Parse dash
+      case line
+        when /^-$/
+          error "Entry content expected"
+        when /^(-\s+)(&\w+.*)/
+          @dash = make_token("-")
+          @pos += $1.size
+          @anchor = make_anchor($2.sub(/#.*/, ""))
+          return
+        when /^(-\s+)(.*)/
+          @dash = make_token("-")
+          @pos += $1.size
+          line = $2
+        when /^-./
+          @pos += 1
+          error "Illegal character after '-'"
+      end
+
+      # Expect key/value pair 
+      case line
+        when /^(\w+)(\s*):\s*$/ # Record
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size + 1
+        when /^(\w+)(\s*:\s*)(&\w+)(\s*)(\S.*)?$/ # Anchor
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size
+          @anchor = make_anchor($3)
+          @pos += $3.size + $4.size
+          $5.nil? or error "Illegal character after anchor"
+        when /^(\w+)(\s*:\s*)(\*.+)(\s*)(\S.*)?/ # Reference
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size
+          @reference = make_reference($3)
+          @pos += $3.size + $4.size
+          $5.nil? or error "Illegal character after reference"
+        when /^(\w+)(\s*:\s*)(["']?.+)/ # Simple value
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size
+          @value = make_value($3)
+        when /^(\w+)(\s*:\s*)(\[.*)/ # Array value
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size
+          litt =~ /.*\]\s*$/ or error "Missing ']'"
+          @value = make_value($3)
+        when /^(\w+)(\s*:\s*)(\{.*)/ # Hash value
+          @ident = make_ident($1)
+          @pos += $1.size + $2.size
+          litt =~ /.*\]\s*$/ or error "Missing '}'"
+          @value = make_value($3)
+        when /^(\*.+)(\s*)(\S.*)?$/
+          @reference = make_reference($1)
+          @pos += $1.size + $2.size
+          $3.nil? or error "Illegal character after reference"
+        when /^\[.*\]\s*$/
+        else
+          @pos += line.size
+          error "Missing value"
+      end
+    end
+
+    # True if the line is the first line of a root table
+    def root_table?() indent == 0 && dash.nil? && value.nil? end
+
+    # True if the line is the first line of a root record
+    def root_record?() indent == 0 && dash.nil? && !value.nil? || false end
+
+    # True if the line is the first line of a table element
+    def element?() !dash.nil? end
+
+    def to_s(long: false)
+      super +
+          if long
+            [ dash && "dash:#{dash.pos}", 
+              ident && "ident:#{ident.pos} #{ident.value}", 
+              value && "value:#{value.pos} #{value.value}",
+              reference && "reference:#{reference.pos} #{reference.value}",
+              anchor && "anchor:##{anchor.pos} #{anchor.value}"
+            ]
+          else
+            [ dash && "dash", 
+              ident && "ident: #{ident.to_s}", 
+              value && "value: #{value.to_s}",
+              reference && "reference: #{reference.to_s}",
+              anchor && "anchor: #{anchor.to_s}"
+            ]
+          end.compact.join(", ")
+    end
+  end
+end
+

data/lib/fixture_fox/parser.rb

@@ -0,0 +1,153 @@
+
+require 'pathname'
+
+module FixtureFox
+  class Parser
+    attr_reader :file
+    attr_reader :ast
+    attr_reader :anchor_files # Name of external anchor files from @anchors directive
+
+    def initialize(file, lines, schema: nil)
+      @file = file 
+      @lines = lines
+      @schema = schema || "public"
+      @anchor_files = []
+    end
+
+    def call(ast = nil)
+      @ast = ast || Ast.new(file)
+
+      # Current schema. The schema is initialized with a synthetic Ident token
+      # because the #analyzer needs a token to emit an error message if the 
+      # public schema doesn't exist
+      schema = Ident.new(file, 1, peek.initial_indent, 1, @schema)
+
+      # Parse
+      get_line
+      while line
+        case line
+          when DirectiveLine
+            case line.directive.value
+              when "schema"
+                schema = line.argument
+                get_line
+              when "include"
+                parse_included_file(line.argument.value)
+                get_line
+              when "anchors" # TODO: Remove. Replaced by a command line option
+                @anchor_files << line.argument.value
+                get_line
+            end
+
+          when Line
+            if line.root_table? && line.empty
+              table = AstTable.new(@ast, schema, line.ident)
+              get_line
+            elsif line.root_table?
+              peek && peek.element? or line.error("Table definition expected")
+              table = AstTable.new(@ast, schema, line.ident)
+              get_line
+              parse_elements(table)
+            elsif line.root_record?
+              peek && peek.indent > 0 or line.error("Record definition expected")
+              # The tokenizer recognizes root records as fields (where the
+              # table name is tokenized as a Value - ie. a text). The following
+              # recreates the table name as a Token and also embeds the record
+              # in an AstTable. This effectively transforms a root-level record
+              # definition into a single-row table definition
+#             table_ident = Ident.new(
+#                 file, line.value.lineno, line.initial_indent, line.value.pos, line.value.value)
+              name = PgGraph.inflector.record_type2table(line.value.value)
+              table_ident = Ident.new(
+                  file, line.value.lineno, line.initial_indent, line.value.pos, name)
+              table = AstTable.new(@ast, schema, table_ident)
+              record = AstRecordElement.new(table, AnchorToken.of_ident(line.ident))
+              get_line
+              parse_members(record, line.indent)
+            else
+              line.error("Table or record definition expected")
+            end
+          else
+            raise "Oops"
+          end
+      end
+
+      @ast
+    end
+
+  private
+    def line() @line end
+    def get_line() @line = @lines.shift end
+    def peek() @lines.first end
+
+    def parse_included_file(path)
+      include_path = Pathname.new(path)
+      if include_path.absolute?
+        include_file = include_path.to_s
+      else
+        including_dir = Pathname.new(file).expand_path.dirname
+        include_file = 
+            Pathname.new(including_dir.to_s + "/" + include_path.to_s)
+            .cleanpath
+            .relative_path_from(Pathname.getwd).to_s
+      end
+      tokenizer = Tokenizer.new(include_file)
+      Parser.new(tokenizer.file, tokenizer.call).call(@ast)
+    end            
+
+    # Parse table elements. Current line should be the first element
+    def parse_elements(table)
+      element_indent = line.indent
+      while line && line.indent == element_indent && line.element?
+        if line.ident.nil? && line.reference
+          AstReferenceElement.new(table, line.reference)
+          get_line
+        elsif line.ident.nil? && line.anchor && !(peek && peek.indent > element_indent)
+          line.error("Empty record definition in table")
+        else
+          if !line.ident && line.anchor # - &label
+            record = AstRecordElement.new(table, line.anchor)
+            indent = line.anchor.pos - 1
+            get_line
+            parse_members(record, indent)
+          else # - key: value
+            record = AstRecordElement.new(table)
+            parse_members(record, line.ident.pos - 1, skip_first_check: true)
+          end
+        end
+      end
+    end
+
+    # Parse record members. Current line should be the first member
+    #
+    # If :skip_first_check the first check of indent is skipped. It is used for
+    # table rows where the first line in a record has a prefixed dash
+    def parse_members(record, indent, skip_first_check: false)
+      constrain indent, Integer
+      while line
+        break if !skip_first_check && line&.indent < indent # Record ends here if line is outdented
+        skip_first_check = false
+        if peek && peek.indent >= indent && peek.element? # This is a table if next line is dashed
+          table = AstTableMember.new(record, line.ident)
+          get_line
+          parse_elements(table)
+        elsif peek && peek.indent > indent # This is a record if next line is indented
+          record_member = AstRecordMember.new(record, line.ident, line.anchor)
+          get_line
+          parse_members(record_member, line.indent)
+        elsif line.anchor && !(peek && peek.indent > indent) # Empty record with a label
+          line.error("Empty record definition")
+        elsif line.anchor && line.dash && !(peek && peek.indent == indent)
+          line.error("Empty record definition 2")
+        elsif line.reference
+          AstReferenceMember.new(record, line.ident, line.reference)
+          get_line
+        else
+          AstFieldMember.new(record, line.ident, line.value || line.reference)
+          get_line
+        end
+      end
+    end
+  end
+end
+

data/lib/fixture_fox/token.rb

@@ -0,0 +1,173 @@
+module FixtureFox
+  class Token
+    attr_reader :file
+    attr_reader :lineno
+    attr_reader :initial_indent
+    attr_reader :pos
+    attr_reader :litt
+    attr_reader :value
+
+    def initialize(file, lineno, initial_indent, pos, litt)
+      @file, @lineno, @initial_indent, @pos, @litt = file, lineno, initial_indent, pos, litt
+      @value = @litt
+    end
+
+    def ==(s) @litt == s end
+    def to_s() @litt end
+
+    def error(msg)
+      raise ParseError.new(file, lineno, initial_indent + pos, msg)
+    end
+  end
+
+  class AnchorToken < Token
+    def initialize(file, lineno, initial_indent, pos, litt)
+      super
+      @value = litt[1..-1].to_sym
+    end
+
+    # Convert an Ident token to a AnchorToken token
+    def self.of_ident(ident)
+      AnchorToken.new(ident.file, ident.lineno, ident.initial_indent, ident.pos, "&#{ident.litt}")
+    end
+  end
+
+  class Empty < Token
+  end
+
+  class Directive < Token
+    def initialize(file, lineno, initial_indent, pos, litt)
+      super
+      @value = litt[1..-1]
+    end
+  end
+
+  class Ident < Token
+  end
+
+  class Reference < Token
+#   attr_reader :reference
+    def initialize(file, lineno, initial_indent, pos, litt)
+      super
+      @value = litt[1..-1]
+    end
+  end
+
+  class Value < Token
+    attr_reader :klass # NilClass, TrueClass, Integer, Float, String, Array
+
+    def initialize(file, lineno, initial_indent, pos, litt)
+      super
+
+      i = 1
+      case litt
+        when /^(nil)/, /^(null)/
+          i += $1.size
+          @klass = NilClass
+          @value = nil
+
+        when /^(true)/, /^(false)/
+          i += $1.size
+          @klass = TrueClass
+          @value = ($1 == "true")
+
+        when /^([-+]?\d+\.\d+)/
+          i += $1.size
+          @klass = Float
+          @value = $1.to_f
+
+        when /^([-+]?\d+)/
+          i += $1.size
+          @klass = Integer
+          @value = $1.to_i
+
+        when /^"/
+          @value = ""
+          @klass = String
+          loop do
+            i < litt.size or self.error("Unterminated string")
+            if litt[i] == "\\"
+              if litt[i+1] == "\\"
+                @value += "\\"
+                i += 2
+              elsif litt[i+1] == '"'
+                @value += '"'
+                i += 2
+              elsif litt[i+1] == 'n'
+                @value += "\n"
+                i += 2
+              elsif litt[i+1] == 't'
+                @value += "\t"
+                i += 2
+              else
+                @value += "\\"
+                i += 1
+              end
+            elsif litt[i] == '"'
+              i += 1
+              break
+            else
+              @value += litt[i]
+              i += 1
+            end
+          end
+
+        when /^'/
+          @value = ""
+          @klass = String
+          loop do
+            i < litt.size or self.error("Unterminated string")
+            if litt[i] == "'"
+              if litt[i+1] == "'"
+                @value += "'"
+                i += 2
+              else
+                i += 1
+                break
+              end
+            else
+              @value += litt[i]
+              i += 1
+            end
+          end
+
+        when /^(\[\s*)(.*)\]$/
+          i += $1.size
+          elements = $2
+          matches = elements.split(/(\s*(?<!(?<!\\{2})\\),\s*)/)
+
+          @value = []
+          while element = matches.shift
+            unescaped_element = element.sub(/\\/, "")
+            @value << Value.new(file, lineno, initial_indent, i, unescaped_element).value
+            i += element.size + (matches.shift&.size || 0)
+          end
+          @klass = Array
+
+        when /^\{/
+          i += litt.size
+          @value = HashParser.parse(litt)
+          @klass = Hash
+
+        else
+          i += litt.size
+          @klass = String
+          @value = litt.dup
+      end
+
+      (litt[i..-1] || "") =~ /^\s*(?:#.*)?$/ or self.error("Illegal characters after literal")
+    end
+
+    def to_s()
+      if klass == NilClass
+        "null"
+#     elsif klass == String
+#       "'#{value.gsub("'", "''")}'"
+      else
+        value.to_s
+      end
+    end
+  end
+
+end
+