csv_plus_plus 0.0.2

data/lib/csv_plus_plus/language/entities/entity.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../entities'
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      # A basic building block of the abstract syntax tree (AST)
+      class Entity
+        attr_reader :id, :type
+
+        # initialize
+        def initialize(type, id: nil)
+          @type = type.to_sym
+          @id = id.downcase.to_sym if id
+        end
+
+        # ==
+        def ==(other)
+          self.class == other.class && @type == other.type && @id == other.id
+        end
+
+        # Respond to predicates that correspond to types like #boolean?, #string?, etc
+        def method_missing(method_name, *_arguments)
+          if method_name =~ /^(\w+)\?$/
+            t = ::Regexp.last_match(1)
+            a_type?(t) && @type == t.to_sym
+          else
+            super
+          end
+        end
+
+        # support predicates by type
+        def respond_to_missing?(method_name, *_arguments)
+          (method_name =~ /^(\w+)\?$/ && a_type?(::Regexp.last_match(1))) || super
+        end
+
+        private
+
+        def a_type?(str)
+          ::CSVPlusPlus::Language::TYPES.include?(str.to_sym)
+        end
+      end
+
+      # An entity that can take arguments
+      class EntityWithArguments < Entity
+        attr_reader :arguments
+
+        # initialize
+        def initialize(type, id: nil, arguments: [])
+          super(type, id:)
+          @arguments = arguments
+        end
+
+        # ==
+        def ==(other)
+          super && @arguments == other.arguments
+        end
+
+        protected
+
+        attr_writer :arguments
+
+        def arguments_to_s
+          @arguments.join(', ')
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/function.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative './entity'
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      # A function definition
+      class Function < EntityWithArguments
+        attr_reader :body
+
+        # Create a function
+        # @param id [Symbool, String] the name of the function - what it will be callable by
+        # @param arguments [Array(Symbol)]
+        # @param body [Entity]
+        def initialize(id, arguments, body)
+          super(:function, id:, arguments: arguments.map(&:to_sym))
+          @body = body
+        end
+
+        # to_s
+        def to_s
+          "def #{@id.to_s.upcase}(#{arguments_to_s}) #{@body}"
+        end
+
+        # ==
+        def ==(other)
+          super && @body == other.body
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/function_call.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      # A function call
+      class FunctionCall < EntityWithArguments
+        # initialize
+        def initialize(id, arguments)
+          super(:function_call, id:, arguments:)
+        end
+
+        # to_s
+        def to_s
+          "#{@id.to_s.upcase}(#{arguments_to_s})"
+        end
+
+        # ==
+        def ==(other)
+          super && @id == other.id
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/number.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      ##
+      # A number value
+      class Number < Entity
+        attr_reader :value
+
+        # initialize
+        def initialize(value)
+          super(:number)
+          @value =
+            if value.instance_of?(::String)
+              value.include?('.') ? Float(value) : Integer(value, 10)
+            else
+              value
+            end
+        end
+
+        # to_s
+        def to_s
+          @value.to_s
+        end
+
+        # ==
+        def ==(other)
+          super && value == other.value
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/runtime_value.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      ##
+      # A runtime value
+      #
+      # These are values which can be materialized at any point via the +resolve_fn+
+      # which takes an ExecutionContext as a param
+      class RuntimeValue < Entity
+        attr_reader :resolve_fn
+
+        # initialize
+        def initialize(resolve_fn)
+          super(:runtime_value)
+          @resolve_fn = resolve_fn
+        end
+
+        # to_s
+        def to_s
+          '(runtime_value)'
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/string.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      ##
+      # A string value
+      class String < Entity
+        attr_reader :value
+
+        # initialize
+        def initialize(value)
+          super(:string)
+          @value = value.gsub(/^"|"$/, '')
+        end
+
+        # to_s
+        def to_s
+          "\"#{@value}\""
+        end
+
+        # ==
+        def ==(other)
+          super && value == other.value
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities/variable.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module CSVPlusPlus
+  module Language
+    module Entities
+      # A reference to a variable
+      class Variable < Entity
+        # initialize
+        def initialize(id)
+          super(:variable, id:)
+        end
+
+        # to_s
+        def to_s
+          "$$#{@id}"
+        end
+
+        # ==
+        def ==(other)
+          super && id == other.id
+        end
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/entities.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative 'entities/boolean'
+require_relative 'entities/cell_reference'
+require_relative 'entities/entity'
+require_relative 'entities/function'
+require_relative 'entities/function_call'
+require_relative 'entities/number'
+require_relative 'entities/runtime_value'
+require_relative 'entities/string'
+require_relative 'entities/variable'
+
+module CSVPlusPlus
+  module Language
+    TYPES = {
+      boolean: ::CSVPlusPlus::Language::Entities::Boolean,
+      cell_reference: ::CSVPlusPlus::Language::Entities::CellReference,
+      function: ::CSVPlusPlus::Language::Entities::Function,
+      function_call: ::CSVPlusPlus::Language::Entities::FunctionCall,
+      number: ::CSVPlusPlus::Language::Entities::Number,
+      runtime_value: ::CSVPlusPlus::Language::Entities::RuntimeValue,
+      string: ::CSVPlusPlus::Language::Entities::String,
+      variable: ::CSVPlusPlus::Language::Entities::Variable
+    }.freeze
+
+    public_constant :TYPES
+  end
+end

data/lib/csv_plus_plus/language/references.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative '../graph'
+require_relative './scope'
+
+module CSVPlusPlus
+  module Language
+    # References in an AST that need to be resolved
+    class References
+      attr_accessor :functions, :variables
+
+      # Extract references from an AST.  And return them in a new +References+ object
+      def self.extract(ast, code_section)
+        new.tap do |refs|
+          ::CSVPlusPlus::Graph.depth_first_search(ast) do |node|
+            next unless node.function_call? || node.variable?
+
+            refs.functions << node if function_reference?(node, code_section)
+            refs.variables << node if node.variable?
+          end
+        end
+      end
+
+      # Is the node a resolvable reference?
+      def self.function_reference?(node, code_section)
+        node.function_call? && (code_section.defined_function?(node.id) || ::BUILTIN_FUNCTIONS.key?(node.id))
+      end
+
+      private_class_method :function_reference?
+
+      # Create an object with empty references.  The caller will build them up as it depth-first-searches
+      def initialize
+        @functions = []
+        @variables = []
+      end
+
+      # are there any references to be resolved?
+      def empty?
+        @functions.empty? && @variables.empty?
+      end
+
+      # to_s
+      def to_s
+        "References(functions: #{@functions}, variables: #{@variables})"
+      end
+
+      # ==
+      def ==(other)
+        @functions == other.functions && @variables == other.variables
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/runtime.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require_relative 'entities'
+require_relative 'syntax_error'
+require 'tempfile'
+
+ENTITIES = ::CSVPlusPlus::Language::Entities
+
+RUNTIME_VARIABLES = {
+  rownum: ::ENTITIES::RuntimeValue.new(->(r) { ::ENTITIES::Number.new(r.row_index + 1) }),
+  cellnum: ::ENTITIES::RuntimeValue.new(->(r) { ::ENTITIES::Number.new(r.cell_index + 1) })
+}.freeze
+
+module CSVPlusPlus
+  module Language
+    ##
+    # The runtime state of the compiler (the current linenumber/row, cell, etc)
+    class Runtime
+      attr_reader :filename, :length_of_code_section, :length_of_csv_section, :length_of_original_file
+
+      attr_accessor :cell, :cell_index, :row_index, :line_number
+
+      # initialize
+      def initialize(input:, filename:)
+        @filename = filename || 'stdin'
+
+        init_input!(input)
+        init!(1)
+      end
+
+      # map over an unparsed file and keep track of line_number and row_index
+      def map_lines(lines, &block)
+        @line_number = 1
+        lines.map do |line|
+          block.call(line).tap { next_line! }
+        end
+      end
+
+      # map over a single row and keep track of the cell and it's index
+      def map_row(row, &block)
+        @cell_index = 0
+        row.map.with_index do |cell, index|
+          set_cell!(cell, index)
+          block.call(cell, index)
+        end
+      end
+
+      # map over all rows and keep track of row and line numbers
+      def map_rows(rows, cells_too: false, &block)
+        @row_index = 0
+        map_lines(rows) do |row|
+          if cells_too
+            # it's either CSV or a Row object
+            map_row((row.is_a?(::CSVPlusPlus::Row) ? row.cells : row), &block)
+          else
+            block.call(row)
+          end
+        end
+      end
+
+      # Increment state to the next line
+      def next_line!
+        @row_index += 1 unless @row_index.nil?
+        @line_number += 1
+      end
+
+      # Set the current cell and index
+      def set_cell!(cell, cell_index)
+        @cell = cell
+        @cell_index = cell_index
+      end
+
+      # Each time we run a parse on the input, call this so that the runtime state
+      # is set to it's default values
+      def init!(start_line_number_at)
+        @row_index = @cell_index = nil
+        @line_number = start_line_number_at
+      end
+
+      # to_s
+      def to_s
+        "Runtime(cell: #{@cell}, row_index: #{@row_index}, cell_index: #{@cell_index})"
+      end
+
+      # get the current (entity) value of a runtime value
+      def runtime_value(var_id)
+        if runtime_variable?(var_id)
+          ::RUNTIME_VARIABLES[var_id.to_sym].resolve_fn.call(self)
+        else
+          raise_syntax_error('Undefined variable', var_id)
+        end
+      end
+
+      # Is +var_id+ a runtime variable?  (it's a static variable otherwise)
+      def runtime_variable?(var_id)
+        ::RUNTIME_VARIABLES.key?(var_id.to_sym)
+      end
+
+      # Called when an error is encoutered during parsing.  It will construct a useful
+      # error with the current +@row/@cell_index+, +@line_number+ and +@filename+
+      def raise_syntax_error(message, bad_input, wrapped_error: nil)
+        raise(::CSVPlusPlus::Language::SyntaxError.new(message, bad_input, self, wrapped_error:))
+      end
+
+      # The currently available input for parsing.  The tmp state will be re-written
+      # between parsing the code section and the CSV section
+      def input
+        @tmp
+      end
+
+      # We mutate the input over and over. It's ok because it's just a Tempfile
+      def rewrite_input!(data)
+        @tmp.truncate(0)
+        @tmp.write(data)
+        @tmp.rewind
+      end
+
+      # Clean up the Tempfile we're using for parsing
+      def cleanup!
+        return unless @tmp
+
+        @tmp.close
+        @tmp.unlink
+        @tmp = nil
+      end
+
+      private
+
+      def count_code_section_lines(lines)
+        eoc = ::CSVPlusPlus::Lexer::END_OF_CODE_SECTION
+        lines.include?(eoc) ? (lines.take_while { |l| l != eoc }).length + 1 : 0
+      end
+
+      def init_input!(input)
+        lines = (input || '').split(/\s*\n\s*/)
+        @length_of_original_file = lines.length
+        @length_of_code_section = count_code_section_lines(lines)
+        @length_of_csv_section = @length_of_original_file - @length_of_code_section
+
+        # we're gonna take our input file, write it to a tmp file then each
+        # step is gonna mutate that tmp file
+        @tmp = ::Tempfile.new
+        rewrite_input!(input)
+      end
+    end
+  end
+end

data/lib/csv_plus_plus/language/scope.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require_relative '../code_section'
+require_relative '../graph'
+require_relative './entities'
+require_relative './references'
+require_relative './syntax_error'
+
+BUILTIN_FUNCTIONS = {
+  # =CELLREF(C) === =INDIRECT(CONCAT($$C, $$rownum))
+  cellref: ::CSVPlusPlus::Language::Entities::Function.new(
+    :cellref,
+    [:cell],
+    ::CSVPlusPlus::Language::Entities::FunctionCall.new(
+      :indirect,
+      [
+        ::CSVPlusPlus::Language::Entities::FunctionCall.new(
+          :concat,
+          [
+            ::CSVPlusPlus::Language::Entities::Variable.new(:cell),
+            ::CSVPlusPlus::Language::Entities::Variable.new(:rownum)
+          ]
+        )
+      ]
+    )
+  )
+}.freeze
+
+module CSVPlusPlus
+  module Language
+    # A class representing the scope of the current Template and responsible for resolving variables
+    # rubocop:disable Metrics/ClassLength
+    class Scope
+      attr_reader :code_section, :runtime
+
+      # initialize with a +Runtime+ and optional +CodeSection+
+      def initialize(runtime:, code_section: nil)
+        @code_section = code_section if code_section
+        @runtime = runtime
+      end
+
+      # Resolve all values in the ast of the current cell being processed
+      def resolve_cell_value
+        return unless (ast = @runtime.cell&.ast)
+
+        last_round = nil
+        loop do
+          refs = ::CSVPlusPlus::Language::References.extract(ast, @code_section)
+          return ast if refs.empty?
+
+          # TODO: throw an error here instead I think - basically we did a round and didn't make progress
+          return ast if last_round == refs
+
+          ast = resolve_functions(resolve_variables(ast, refs.variables), refs.functions)
+        end
+      end
+
+      # Set the +code_section+ and resolve all inner dependencies in it's variables and functions.
+      def code_section=(code_section)
+        @code_section = code_section
+
+        resolve_static_variables!
+        resolve_static_functions!
+      end
+
+      # to_s
+      def to_s
+        "Scope(code_section: #{@code_section}, runtime: #{@runtime})"
+      end
+
+      private
+
+      # Resolve all variable references defined statically in the code section
+      def resolve_static_variables!
+        variables = @code_section.variables
+        last_var_dependencies = {}
+        # TODO: might not need the infinite loop wrap
+        loop do
+          var_dependencies, resolution_order = variable_resolution_order(only_static_vars(variables))
+          return if var_dependencies == last_var_dependencies
+
+          # TODO: make the contract better here where we're not seting the variables of another class
+          @code_section.variables = resolve_dependencies(var_dependencies, resolution_order, variables)
+          last_var_dependencies = var_dependencies.clone
+        end
+      end
+
+      def only_static_vars(var_dependencies)
+        var_dependencies.reject { |k| @runtime.runtime_variable?(k) }
+      end
+
+      # Resolve all functions defined statically in the code section
+      def resolve_static_functions!
+        # TODO: I'm still torn if it's worth replacing function references
+        #
+        # my current theory is that if we resolve static functions befor processing each cell,
+        # overall compile time will be improved because there will be less to do for each cell
+      end
+
+      def resolve_functions(ast, refs)
+        refs.reduce(ast.dup) do |acc, elem|
+          function_replace(acc, elem.id, resolve_function(elem.id))
+        end
+      end
+
+      def resolve_variables(ast, refs)
+        refs.reduce(ast.dup) do |acc, elem|
+          variable_replace(acc, elem.id, resolve_variable(elem.id))
+        end
+      end
+
+      # Make a copy of the AST represented by +node+ and replace +fn_id+ with +replacement+ throughout
+      def function_replace(node, fn_id, replacement)
+        if node.function_call? && node.id == fn_id
+          apply_arguments(replacement, node)
+        elsif node.function_call?
+          arguments = node.arguments.map { |n| function_replace(n, fn_id, replacement) }
+          ::CSVPlusPlus::Language::Entities::FunctionCall.new(node.id, arguments)
+        else
+          node
+        end
+      end
+
+      def resolve_function(fn_id)
+        id = fn_id.to_sym
+        return @code_section.functions[id] if @code_section.defined_function?(id)
+
+        # this will throw a syntax error if it doesn't exist (which is what we want)
+        return ::BUILTIN_FUNCTIONS[id] if ::BUILTIN_FUNCTIONS.key?(id)
+
+        @runtime.raise_syntax_error('Unknown function', fn_id)
+      end
+
+      def apply_arguments(function, function_call)
+        i = 0
+        function.arguments.reduce(function.body.dup) do |ast, argument|
+          variable_replace(ast, argument, function_call.arguments[i]).tap do
+            i += 1
+          end
+        end
+      end
+
+      # Make a copy of the AST represented by +node+ and replace +var_id+ with +replacement+ throughout
+      def variable_replace(node, var_id, replacement)
+        if node.function_call?
+          arguments = node.arguments.map { |n| variable_replace(n, var_id, replacement) }
+          ::CSVPlusPlus::Language::Entities::FunctionCall.new(node.id, arguments)
+        elsif node.variable? && node.id == var_id
+          replacement
+        else
+          node
+        end
+      end
+
+      def resolve_variable(var_id)
+        id = var_id.to_sym
+        return @code_section.variables[id] if @code_section.defined_variable?(id)
+
+        # this will throw a syntax error if it doesn't exist (which is what we want)
+        @runtime.runtime_value(id)
+      end
+
+      def check_unbound_vars(dependencies, variables)
+        unbound_vars = dependencies.values.flatten - variables.keys
+        return if unbound_vars.empty?
+
+        @runtime.raise_syntax_error('Undefined variables', unbound_vars.map(&:to_s).join(', '))
+      end
+
+      def variable_resolution_order(variables)
+        # we have a hash of variables => ASTs but they might have references to each other, so
+        # we need to interpolate them first (before interpolating the cell values)
+        var_dependencies = ::CSVPlusPlus::Graph.dependency_graph(variables, @runtime)
+        # are there any references that we don't have variables for? (undefined variable)
+        check_unbound_vars(var_dependencies, variables)
+
+        # a topological sort will give us the order of dependencies
+        [var_dependencies, ::CSVPlusPlus::Graph.topological_sort(var_dependencies)]
+        # TODO: don't expose this exception directly to the caller
+      rescue ::TSort::Cyclic
+        @runtime.raise_syntax_error('Cyclic variable dependency detected', var_refs.keys)
+      end
+
+      def resolve_dependencies(var_dependencies, resolution_order, variables)
+        {}.tap do |resolved_vars|
+          # for each var and each dependency it has, build up and mutate resolved_vars
+          resolution_order.each do |var|
+            resolved_vars[var] = variables[var].dup
+
+            var_dependencies[var].each do |dependency|
+              resolved_vars[var] = variable_replace(resolved_vars[var], dependency, variables[dependency])
+            end
+          end
+        end
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end