forthic 0.2.4 → 0.3.0

data/lib/forthic/interpreter.rb

@@ -1,518 +1,877 @@
 # frozen_string_literal: true
 
-require_relative "forthic_error"
-require_relative "errors/unknown_word_error"
-require_relative "tokenizer"
-require_relative "token"
-require_relative "code_location"
-require_relative "positioned_string"
-require_relative "words/word"
-require_relative "words/push_value_word"
-require_relative "words/start_module_word"
-require_relative "words/end_module_word"
-require_relative "words/end_array_word"
-require_relative "words/definition_word"
-require_relative "global_module"
+require_relative 'tokenizer'
+require_relative 'module'
+require_relative 'errors'
+require_relative 'literals'
+require 'time'
 
 module Forthic
-  # Error codes used throughout the interpreter
-  module ErrorCodes
-    SCREEN_NOT_FOUND = "screen-not-found"
-    EXECUTION_ERROR = "execution-error"
-    MODULE_NOT_FOUND = "module-not-found"
-    STACK_UNDERFLOW = "stack-underflow"
-    MODULE_EXECUTION_ERROR = "module-execution-error"
-    UNKNOWN_TOKEN = "unknown-token"
-    NESTED_DEFINITION = "nested-definition"
-    NESTED_MEMO_DEFINITION = "nested-memo-definition"
-    DEFINITION_WITHOUT_START = "definition-without-start"
-    MISSING_DEFINITION = "missing-definition"
-    WORD_NOT_FOUND = "word-not-found"
+  # Timestamp - Struct for profiling timestamps
+  Timestamp = Struct.new(:label, :time_ms, keyword_init: true)
+
+  # StartModuleWord - Handles module creation and switching
+  #
+  # Pushes a module onto the module stack, creating it if necessary.
+  # An empty name refers to the app module.
+  class StartModuleWord < Word
+    def execute(interp)
+      # The app module is the only module with a blank name
+      if @name == ""
+        interp.module_stack_push(interp.get_app_module)
+        return
+      end
+
+      # If the module is used by the current module, push it onto the stack, otherwise
+      # create a new module.
+      mod = interp.cur_module.find_module(@name)
+      unless mod
+        mod = Module.new(@name)
+        interp.cur_module.register_module(mod.name, mod.name, mod)
+
+        # If we're at the app module, also register with interpreter
+        if interp.cur_module.name == ""
+          interp.register_module(mod)
+        end
+      end
+      interp.module_stack_push(mod)
+    end
   end
 
-  # Manages execution state for the interpreter
-  class ExecutionState
-    attr_accessor :stack, :module_stack, :is_compiling, :should_stop,
-      :is_memo_definition, :cur_definition, :string_location
+  # EndModuleWord - Pops the current module from the module stack
+  #
+  # Completes module context and returns to the previous module.
+  class EndModuleWord < Word
+    def initialize
+      super("}")
+    end
 
-    def initialize(app_module)
-      @stack = []
-      @module_stack = [app_module]
-      @is_compiling = false
-      @should_stop = false
-      @is_memo_definition = false
-      @cur_definition = nil
-      @string_location = nil
+    def execute(interp)
+      interp.module_stack_pop
     end
+  end
 
-    def reset(app_module)
-      @stack = []
-      @module_stack = [app_module]
-      @is_compiling = false
-      @is_memo_definition = false
-      @cur_definition = nil
-      @string_location = nil
+  # EndArrayWord - Collects items from stack into an array
+  #
+  # Pops items from the stack until a START_ARRAY token is found,
+  # then pushes them as a single array in the correct order.
+  class EndArrayWord < Word
+    def initialize
+      super("]")
+    end
+
+    def execute(interp)
+      items = []
+      item = interp.stack_pop
+      # NOTE: This won't infinite loop because interp.stack_pop will eventually fail
+      loop do
+        break if item.is_a?(Token) && item.type == TokenType::START_ARRAY
+        items << item
+        item = interp.stack_pop
+      end
+      items.reverse!
+      interp.stack_push(items)
     end
   end
 
-  # Manages profiling state and operations
-  class ProfilingState
-    attr_reader :word_counts, :is_profiling, :start_profile_time, :timestamps
+  # Stack - Wrapper for the interpreter's data stack
+  #
+  # Provides stack operations with support for array indexing.
+  # Handles PositionedString unwrapping and provides JSON serialization.
+  # Items can be accessed with bracket notation (e.g., stack[0]).
+  class Stack
+    def initialize(items = [])
+      @items = items
+    end
 
-    def initialize
-      @word_counts = {}
-      @is_profiling = false
-      @start_profile_time = nil
-      @timestamps = []
+    # Array-like access methods
+    def [](index)
+      @items[index]
     end
 
-    def start_profiling
-      @is_profiling = true
-      @timestamps = []
-      @start_profile_time = Time.now
-      add_timestamp("START")
-      @word_counts = {}
+    def []=(index, value)
+      @items[index] = value
     end
 
-    def stop_profiling
-      add_timestamp("END")
-      @is_profiling = false
+    def length
+      @items.length
     end
 
-    def count_word(word)
-      return unless @is_profiling
-      @word_counts[word.name] ||= 0
-      @word_counts[word.name] += 1
+    alias size length
+
+    def get_items
+      @items.map do |item|
+        if item.is_a?(PositionedString)
+          item.value
+        else
+          item
+        end
+      end
     end
 
-    def add_timestamp(label)
-      return unless @is_profiling
-      timestamp = {label: label, time_ms: (Time.now - @start_profile_time) * 1000}
-      @timestamps.push(timestamp)
+    def get_raw_items
+      @items
     end
 
-    def word_histogram
-      @word_counts.map { |name, count| {word: name, count: count} }.sort_by { |item| -item[:count] }
+    def set_raw_items(items)
+      @items = items
+    end
+
+    def to_json(*args)
+      @items.to_json(*args)
+    end
+
+    def pop
+      @items.pop
+    end
+
+    def push(item)
+      @items.push(item)
+    end
+
+    # Duplicate stack with a shallow copy of items
+    def dup
+      Stack.new(@items.dup)
     end
   end
 
+  # Interpreter - Base Forthic interpreter
+  #
+  # Core interpreter that tokenizes and executes Forthic code.
+  # Manages the data stack, module stack, and execution context.
+  #
+  # Features:
+  # - Stack-based execution model
+  # - Module system with imports and namespacing
+  # - Literal handlers for parsing values (numbers, dates, booleans, etc.)
+  # - Error handling with recovery attempts
+  # - Profiling and performance tracking
+  # - Streaming execution support
+  #
+  # Note: This is the base interpreter without standard library modules.
+  # Use StandardInterpreter for a full-featured interpreter with stdlib.
   class Interpreter
-    # Core interpreter components
-    attr_reader :global_module, :app_module, :registered_modules
-    # Screen and module management
-    attr_accessor :screens, :default_module_flags, :module_flags
-    # State objects
-    attr_reader :execution_state, :profiling_state
-
-    # Token handler lookup table
-    TOKEN_HANDLERS = {
-      TokenType::STRING => :handle_string_token,
-      TokenType::COMMENT => :handle_comment_token,
-      TokenType::START_ARRAY => :handle_start_array_token,
-      TokenType::END_ARRAY => :handle_end_array_token,
-      TokenType::START_MODULE => :handle_start_module_token,
-      TokenType::END_MODULE => :handle_end_module_token,
-      TokenType::START_DEF => :handle_start_definition_token,
-      TokenType::START_MEMO => :handle_start_memo_token,
-      TokenType::END_DEF => :handle_end_definition_token,
-      TokenType::WORD => :handle_word_token
-    }.freeze
+    attr_reader :timezone, :stack
+    attr_accessor :handle_error, :max_attempts, :on_word_execute
 
-    def initialize
+    def initialize(modules = [], timezone = "UTC")
+      @timezone = timezone
+      @stack = Stack.new
+
+      @tokenizer_stack = []
+      @max_attempts = 3
+      @handle_error = nil
+
+      @app_module = Module.new("")
+      @app_module.set_interp(self)
+      @module_stack = [@app_module]
       @registered_modules = {}
-      @screens = {}
-      @default_module_flags = {}
-      @module_flags = {}
+      @is_compiling = false
+      @is_memo_definition = false
+      @cur_definition = nil
 
-      @global_module = GlobalModule.new(self)
-      @app_module = ForthicModule.new("", self)
+      # Debug support
+      @string_location = nil
+      @previous_token = nil
 
-      @execution_state = ExecutionState.new(@app_module)
-      @profiling_state = ProfilingState.new
-    end
+      # Profiling support
+      @word_counts = {}
+      @is_profiling = false
+      @start_profile_time = nil
+      @timestamps = []
+
+      # Streaming support
+      @streaming_token_index = 0
+      @stream = false
+      @previous_delta_length = 0
+
+      # Progress callback for streaming execution
+      @on_word_execute = nil
+      @word_execution_count = 0
+      @total_words_estimate = 0
 
-    def halt
-      @execution_state.should_stop = true
+      # Literal handlers
+      @literal_handlers = []
+      register_standard_literals
+
+      # If modules are provided, import them unprefixed as a convenience
+      import_modules(modules)
     end
 
-    # @return [ForthicModule]
-    def get_app_module
-      @app_module
+    def get_timezone
+      @timezone
     end
 
-    # @return [CodeLocation, nil]
-    def get_string_location
-      @execution_state.string_location
+    def set_timezone(timezone)
+      @timezone = timezone
     end
 
-    # Delegation methods for execution state
-    def stack
-      @execution_state.stack
+    def get_app_module
+      @app_module
     end
 
-    def stack=(new_stack)
-      @execution_state.stack = new_stack
+    def get_top_input_string
+      return "" if @tokenizer_stack.empty?
+      @tokenizer_stack[0].get_input_string
     end
 
-    def module_stack
-      @execution_state.module_stack
+    def get_tokenizer
+      @tokenizer_stack.last
     end
 
-    def is_compiling
-      @execution_state.is_compiling
+    def get_string_location
+      @string_location
     end
 
-    def cur_definition
-      @execution_state.cur_definition
+    def set_max_attempts(max_attempts)
+      @max_attempts = max_attempts
     end
 
-    # @param [String] module_id
-    # @param [Hash] flags
-    def set_flags(module_id, flags)
-      @default_module_flags[module_id] = flags
-      @module_flags[module_id] = flags
+    def set_error_handler(handle_error)
+      @handle_error = handle_error
     end
 
-    # @param [String] module_id
-    # @return [Hash]
-    def get_flags(module_id)
-      module_flags = @module_flags[module_id] || {}
-      result = module_flags.dup
-      @module_flags[module_id] = @default_module_flags[module_id].dup
-      result
+    def get_max_attempts
+      @max_attempts
     end
 
-    # @param [String] module_id
-    # @param [Hash] flags
-    def modify_flags(module_id, flags)
-      module_flags = @module_flags[module_id] || {}
-      @module_flags[module_id] = module_flags.merge(flags)
+    def get_error_handler
+      @handle_error
     end
 
     def reset
+      @stack = Stack.new
       @app_module.variables = {}
-      @execution_state.reset(@app_module)
-    end
 
-    # @param [String] screen_name
-    # @return [String]
-    def get_screen_forthic(screen_name)
-      screen = @screens[screen_name]
-      raise ForthicError.new(ErrorCodes::SCREEN_NOT_FOUND, "Unable to find screen \"#{screen_name}\"", "Screen not found. Check the screen name for typos or ensure it has been properly registered.") unless screen
-      screen
+      @module_stack = [@app_module]
+      @is_compiling = false
+      @is_memo_definition = false
+      @cur_definition = nil
+
+      # Debug support
+      @string_location = nil
     end
 
-    # @param [String] string
-    # @param [CodeLocation, nil] reference_location
-    # @return [Boolean]
     def run(string, reference_location = nil)
-      tokenizer = Tokenizer.new(string, reference_location)
-      run_with_tokenizer(tokenizer)
+      @tokenizer_stack.push(Tokenizer.new(string, reference_location))
+
+      if @handle_error
+        execute_with_recovery
+      else
+        run_with_tokenizer(@tokenizer_stack.last)
+      end
+
+      @tokenizer_stack.pop
+      true
+    end
+
+    def execute_with_recovery(num_attempts = 0)
+      num_attempts += 1
+      if num_attempts > @max_attempts
+        raise TooManyAttemptsError.new(
+          get_top_input_string,
+          num_attempts,
+          @max_attempts
+        )
+      end
+      continue_execution
+      num_attempts
+    rescue => e
+      raise unless @handle_error
+      @handle_error.call(e, self)
+      execute_with_recovery(num_attempts)
+    end
+
+    def continue_execution
+      run_with_tokenizer(@tokenizer_stack.last)
     end
 
-    # @param [Tokenizer] tokenizer
-    # @return [Boolean]
     def run_with_tokenizer(tokenizer)
       token = nil
       loop do
+        @previous_token = token
         token = tokenizer.next_token
         handle_token(token)
-        break if token.type == TokenType::EOS || @execution_state.should_stop
-        next if [TokenType::START_DEF, TokenType::END_DEF, TokenType::COMMENT].include?(token.type) || @execution_state.is_compiling
+        break if token.type == TokenType::EOS
       end
-      true
-    rescue ForthicError => e
-      # Re-raise ForthicError and its subclasses without wrapping
-      raise e
-    rescue => e
-      # Wrap non-ForthicError exceptions
-      error = ForthicError.new(ErrorCodes::EXECUTION_ERROR, "Error executing token '#{token&.string}'", "An unexpected error occurred during execution. Check the token syntax and try again.", token&.location)
-      error.set_caught_error(e)
-      raise error
+      true # Done executing
     end
 
-    # @return [ForthicModule]
     def cur_module
-      @execution_state.module_stack.last
+      @module_stack.last
     end
 
-    # @param [String] name
-    # @return [ForthicModule]
     def find_module(name)
-      raise ArgumentError, "Module name cannot be nil" if name.nil?
-      raise ArgumentError, "Module name cannot be empty" if name.empty?
-
       result = @registered_modules[name]
-      raise ForthicError.new(ErrorCodes::MODULE_NOT_FOUND, "Module '#{name}' not found", "Check the module name for typos and ensure it has been properly registered.") unless result
+      unless result
+        raise UnknownModuleError.new(
+          get_top_input_string,
+          name,
+          location: @string_location
+        )
+      end
+      result
+    end
+
+    def stack_peek
+      top = @stack[@stack.length - 1]
+      result = top
+      result = top.value if top.is_a?(PositionedString)
       result
     end
 
-    # @param [Object] val
     def stack_push(val)
-      @execution_state.stack.push(val)
+      @stack.push(val)
     end
 
-    # @return [Object]
     def stack_pop
-      raise ForthicError.new(ErrorCodes::STACK_UNDERFLOW, "Stack underflow", "Attempted to pop from an empty stack. This indicates a logical error in the Forthic code.") if @execution_state.stack.empty?
-      result = @execution_state.stack.pop
-      @execution_state.string_location = result.is_a?(PositionedString) ? result.location : nil
-      result.is_a?(PositionedString) ? result.value_of : result
+      if @stack.length.zero?
+        tokenizer = @tokenizer_stack.length > 0 ? get_tokenizer : nil
+        raise StackUnderflowError.new(
+          get_top_input_string,
+          location: tokenizer&.get_token_location
+        )
+      end
+      result = @stack.pop
+
+      # If we have a PositionedString, we need to record the location
+      @string_location = nil
+      if result.is_a?(PositionedString)
+        positioned_string = result
+        result = positioned_string.value
+        @string_location = positioned_string.location
+      end
+      result
+    end
+
+    def get_stack
+      @stack
+    end
+
+    def set_stack(stack)
+      @stack = stack
     end
 
-    # @param [ForthicModule] mod
     def module_stack_push(mod)
-      raise ArgumentError, "Module cannot be nil" if mod.nil?
-      @execution_state.module_stack.push(mod)
+      @module_stack.push(mod)
     end
 
     def module_stack_pop
-      @execution_state.module_stack.pop
+      @module_stack.pop
     end
 
-    # @param [ForthicModule] mod
     def register_module(mod)
-      raise ArgumentError, "Module cannot be nil" if mod.nil?
-      raise ArgumentError, "Module must respond to :name" unless mod.respond_to?(:name)
       @registered_modules[mod.name] = mod
+      mod.set_interp(self)
+    end
+
+    # If names is an array of strings, import each module without a prefix (empty string)
+    # If names is an array of arrays, import each module using the first element as the
+    # module name and the second element as the prefix
+    def use_modules(names)
+      names.each do |name|
+        module_name = name
+        prefix = "" # Default to empty prefix (no prefix)
+        if name.is_a?(Array)
+          module_name = name[0]
+          prefix = name[1] # Allow explicit prefix specification
+        end
+        mod = find_module(module_name)
+        get_app_module.import_module(prefix, mod, self)
+      end
     end
 
-    # @param [ForthicModule] mod
-    # @param [String] prefix
+    # A convenience method to register and use a module
     def import_module(mod, prefix = "")
-      raise ArgumentError, "Module cannot be nil" if mod.nil?
       register_module(mod)
-      @app_module.import_module(prefix, mod, self)
+      use_modules([[mod.name, prefix]])
+    end
+
+    def import_modules(modules)
+      modules.each do |mod|
+        import_module(mod)
+      end
+    end
+
+    # Transforms simple module names to unprefixed imports: "math" -> ["math", ""]
+    # Preserves explicit prefix specifications: ["math", "m"] -> ["math", "m"]
+    def use_modules_unprefixed(names)
+      unprefixed = names.map do |name|
+        name.is_a?(Array) ? name : [name, ""]
+      end
+      use_modules(unprefixed)
     end
 
-    # @param [ForthicModule] mod
     def run_module_code(mod)
-      raise ArgumentError, "Module cannot be nil" if mod.nil?
       module_stack_push(mod)
-      run(mod.forthic_code)
+      begin
+        # Set source to module name when running module code
+        module_location = CodeLocation.new(source: mod.name)
+        run(mod.forthic_code, module_location)
+      rescue => e
+        raise ModuleError.new(
+          get_top_input_string,
+          mod.name,
+          e,
+          location: @string_location,
+          cause: e
+        )
+      end
+
       module_stack_pop
-    rescue => e
-      error = ForthicError.new(ErrorCodes::MODULE_EXECUTION_ERROR, "Error executing module '#{mod.name}'", "An error occurred while running the module code. Check the module implementation for syntax errors.")
-      error.set_caught_error(e)
-      raise error
     end
 
-    # @param [String] name
-    # @return [Word, nil]
+    # ======================
+    # Literal Handlers
+
+    # Register standard literal handlers
+    # Order matters: more specific handlers first
+    def register_standard_literals
+      @literal_handlers = [
+        method(:to_bool).to_proc,                                  # TRUE, FALSE
+        method(:to_float).to_proc,                                 # 3.14
+        Forthic.to_zoned_datetime(@timezone),                      # 2020-06-05T10:15:00Z
+        Forthic.to_literal_date(@timezone),                        # 2020-06-05, YYYY-MM-DD
+        method(:to_time).to_proc,                                  # 9:00, 11:30 PM
+        method(:to_int).to_proc,                                   # 42
+      ]
+    end
+
+    # Register a custom literal handler
+    # Handlers are checked in registration order
+    def register_literal_handler(handler)
+      @literal_handlers << handler
+    end
+
+    # Unregister a literal handler
+    def unregister_literal_handler(handler)
+      @literal_handlers.delete(handler)
+    end
+
+    # Try to parse string as a literal value
+    # Returns PushValueWord if successful, nil otherwise
+    def find_literal_word(name)
+      @literal_handlers.each do |handler|
+        value = handler.call(name)
+        return PushValueWord.new(name, value) unless value.nil?
+      end
+      nil
+    end
+
+    # ======================
+    # Find Word
+
     def find_word(name)
+      # 1. Check module stack (dictionary words + variables)
       result = nil
-      @execution_state.module_stack.reverse_each do |m|
+      (@module_stack.length - 1).downto(0) do |i|
+        m = @module_stack[i]
         result = m.find_word(name)
         break if result
       end
-      result ||= @global_module.find_word(name)
+
+      # 2. Check literal handlers as fallback
+      result = find_literal_word(name) unless result
+
+      # 3. Throw error if still not found
+      unless result
+        raise UnknownWordError.new(
+          get_top_input_string,
+          name,
+          location: get_string_location
+        )
+      end
+
       result
     end
 
-    # Delegation methods for profiling
+    # ======================
+    # Profiling
+
     def start_profiling
-      @profiling_state.start_profiling
+      @is_profiling = true
+      @timestamps = []
+      @start_profile_time = Time.now.to_f * 1000 # milliseconds
+      add_timestamp("START")
+      @word_counts = {}
     end
 
-    def stop_profiling
-      @profiling_state.stop_profiling
+    def count_word(word)
+      return unless @is_profiling
+      name = word.name
+      @word_counts[name] ||= 0
+      @word_counts[name] += 1
     end
 
-    def count_word(word)
-      @profiling_state.count_word(word)
+    def stop_profiling
+      add_timestamp("END")
+      @is_profiling = false
     end
 
     def add_timestamp(label)
-      @profiling_state.add_timestamp(label)
+      return unless @is_profiling
+      timestamp = Timestamp.new(
+        label: label,
+        time_ms: (Time.now.to_f * 1000) - @start_profile_time
+      )
+      @timestamps << timestamp
     end
 
     def word_histogram
-      @profiling_state.word_histogram
+      items = []
+      @word_counts.each do |name, count|
+        items << { word: name, count: count }
+      end
+      items.sort_by { |item| -item[:count] }
     end
 
     def profile_timestamps
-      @profiling_state.timestamps
+      @timestamps
     end
 
-    # @param [Token] token
-    def handle_token(token)
-      return if token.type == TokenType::EOS
+    # ======================
+    # Handle tokens
 
-      handler = TOKEN_HANDLERS[token.type]
-      if handler
-        send(handler, token)
+    def handle_token(token)
+      case token.type
+      when TokenType::STRING
+        handle_string_token(token)
+      when TokenType::COMMENT
+        handle_comment_token(token)
+      when TokenType::START_ARRAY
+        handle_start_array_token(token)
+      when TokenType::END_ARRAY
+        handle_end_array_token(token)
+      when TokenType::START_MODULE
+        handle_start_module_token(token)
+      when TokenType::END_MODULE
+        handle_end_module_token(token)
+      when TokenType::START_DEF
+        handle_start_definition_token(token)
+      when TokenType::START_MEMO
+        handle_start_memo_token(token)
+      when TokenType::END_DEF
+        handle_end_definition_token(token)
+      when TokenType::DOT_SYMBOL
+        handle_dot_symbol_token(token)
+      when TokenType::WORD
+        handle_word_token(token)
+      when TokenType::EOS
+        if @is_compiling
+          raise MissingSemicolonError.new(
+            get_top_input_string,
+            location: @previous_token&.location
+          )
+        end
       else
-        raise ForthicError.new(ErrorCodes::UNKNOWN_TOKEN, "Unknown token type '#{token.string}'", "This token type is not recognized. Check for typos or unsupported syntax.", token.location)
+        raise UnknownTokenError.new(
+          get_top_input_string,
+          token.string,
+          location: @string_location
+        )
       end
     end
 
-    # @param [Token] token
     def handle_string_token(token)
       value = PositionedString.new(token.string, token.location)
       handle_word(PushValueWord.new("<string>", value))
     end
 
-    # @param [Token] token
+    def handle_dot_symbol_token(token)
+      value = PositionedString.new(token.string, token.location)
+      handle_word(PushValueWord.new("<dot-symbol>", value))
+    end
+
+    # Start/end module tokens are treated as IMMEDIATE words *and* are also compiled
     def handle_start_module_token(token)
       word = StartModuleWord.new(token.string)
-      @execution_state.cur_definition.add_word(word) if @execution_state.is_compiling
-      count_word(word)
+
+      @cur_definition.add_word(word) if @is_compiling
+      count_word(word) # For profiling
       word.execute(self)
     end
 
-    # @param [Token] _token
     def handle_end_module_token(_token)
       word = EndModuleWord.new
-      @execution_state.cur_definition.add_word(word) if @execution_state.is_compiling
+
+      @cur_definition.add_word(word) if @is_compiling
       count_word(word)
       word.execute(self)
     end
 
-    # @param [Token] token
     def handle_start_array_token(token)
       handle_word(PushValueWord.new("<start_array_token>", token))
     end
 
-    # @param [Token] _token
     def handle_end_array_token(_token)
       handle_word(EndArrayWord.new)
     end
 
-    # @param [Token] _token
     def handle_comment_token(_token)
-      # Handle comment token (no-op)
+      # Comment handling (currently no-op)
     end
 
-    # @param [Token] token
     def handle_start_definition_token(token)
-      raise ForthicError.new(ErrorCodes::NESTED_DEFINITION, "Nested definition not allowed", "A definition was started while another definition is active. Ensure all definitions end with semicolons.", token.location) if @execution_state.is_compiling
-      @execution_state.cur_definition = DefinitionWord.new(token.string)
-      @execution_state.is_compiling = true
-      @execution_state.is_memo_definition = false
+      if @is_compiling
+        raise MissingSemicolonError.new(
+          get_top_input_string,
+          location: @previous_token&.location
+        )
+      end
+      @cur_definition = DefinitionWord.new(token.string)
+      @is_compiling = true
+      @is_memo_definition = false
     end
 
-    # @param [Token] token
     def handle_start_memo_token(token)
-      raise ForthicError.new(ErrorCodes::NESTED_MEMO_DEFINITION, "Nested memo definition not allowed", "A memo definition was started while another definition is active. Ensure all definitions end with semicolons.", token.location) if @execution_state.is_compiling
-      @execution_state.cur_definition = DefinitionWord.new(token.string)
-      @execution_state.is_compiling = true
-      @execution_state.is_memo_definition = true
+      if @is_compiling
+        raise MissingSemicolonError.new(
+          get_top_input_string,
+          location: @previous_token&.location
+        )
+      end
+      @cur_definition = DefinitionWord.new(token.string)
+      @is_compiling = true
+      @is_memo_definition = true
     end
 
-    # @param [Token] token
     def handle_end_definition_token(token)
-      raise ForthicError.new(ErrorCodes::DEFINITION_WITHOUT_START, "Definition ended without start", "A definition was ended when none was active. Check for extra semicolons.", token.location) unless @execution_state.is_compiling
-      raise ForthicError.new(ErrorCodes::MISSING_DEFINITION, "No current definition to end", "Internal error: definition state is inconsistent.", token.location) unless @execution_state.cur_definition
-      if @execution_state.is_memo_definition
-        cur_module.add_memo_words(@execution_state.cur_definition)
+      unless @is_compiling && @cur_definition
+        raise ExtraSemicolonError.new(
+          get_top_input_string,
+          location: token.location
+        )
+      end
+
+      if @is_memo_definition
+        cur_module.add_memo_words(@cur_definition)
       else
-        cur_module.add_word(@execution_state.cur_definition)
+        cur_module.add_word(@cur_definition)
       end
-      @execution_state.is_compiling = false
+      @is_compiling = false
     end
 
-    # @param [Token] token
     def handle_word_token(token)
-      word = find_word(token.string)
-      unless word
-        # Generate suggested words for better error messages
-        suggested_words = find_similar_words(token.string)
-        raise Errors::UnknownWordError.new(token.string, token.location, suggested_words)
-      end
+      word = find_word(token.string) # Throws UnknownWordError if not found
       handle_word(word, token.location)
     end
 
-    # @param [Word] word
-    # @param [CodeLocation, nil] location
     def handle_word(word, location = nil)
-      if @execution_state.is_compiling
+      if @is_compiling
         word.set_location(location)
-        @execution_state.cur_definition.add_word(word)
+        @cur_definition.add_word(word)
       else
         count_word(word)
+
+        # Notify progress callback if set (for streaming execution)
+        if @on_word_execute && word.is_a?(DefinitionWord)
+          @word_execution_count += 1
+          @on_word_execute.call(word.name, @word_execution_count, @total_words_estimate)
+        end
+
         word.execute(self)
       end
     end
 
-    private
+    # Estimate the number of words in code (for progress tracking)
+    def estimate_word_count(code)
+      # Simple heuristic: count uppercase words (likely Forthic words)
+      code.scan(/[A-Z][A-Z0-9_-]*/).length
+    end
 
-    # Find words similar to the given word name for error suggestions
-    # @param [String] word_name The unknown word to find suggestions for
-    # @return [Array<String>] Array of similar word names
-    def find_similar_words(word_name)
-      return [] if word_name.nil? || word_name.empty?
+    # Set up progress tracking for streaming execution
+    def setup_progress_tracking(code)
+      @word_execution_count = 0
+      @total_words_estimate = estimate_word_count(code)
+    end
 
-      all_words = collect_available_words
+    # Reset progress tracking
+    def reset_progress_tracking
+      @word_execution_count = 0
+      @total_words_estimate = 0
+      @on_word_execute = nil
+    end
 
-      # Find words with similar names using simple string distance
-      suggestions = all_words.select do |available_word|
-        levenshtein_distance(word_name.downcase, available_word.downcase) <= 2
-      end
+    # Streaming execution support
+    def streaming_run(code_stream, done, reference_location = nil)
+      # Create a new Tokenizer for the full string
+      tokenizer = Tokenizer.new(code_stream, reference_location, done ? false : true)
+      tokens = []
+      eos_found = false
 
-      # If no close matches, try prefix matching
-      if suggestions.empty?
-        suggestions = all_words.select do |available_word|
-          available_word.downcase.start_with?(word_name[0, 2].downcase) ||
-            word_name.downcase.start_with?(available_word[0, 2].downcase)
+      @tokenizer_stack.push(tokenizer)
+
+      # Gather tokens from the beginning
+      loop do
+        token = tokenizer.next_token
+        break unless token
+
+        # If we hit an EOS token then push it and break
+        if token.type == TokenType::EOS
+          tokens << token
+          eos_found = true
+          break
         end
+
+        tokens << token
       end
 
-      suggestions.take(3)
-    end
+      delta = eos_found ? nil : tokenizer.get_string_delta
 
-    # Collect all available word names from all modules in scope
-    # @return [Array<String>] Array of all available word names
-    def collect_available_words
-      words = []
+      new_stop = find_last_word_or_eos(tokens)
 
-      # Collect from global module (hardcoded common words since GlobalModule uses methods)
-      global_words = %w[
-        POP DUP SWAP >STR CONCAT SPLIT JOIN /N /R /T LOWERCASE UPPERCASE
-        APPEND REVERSE UNIQUE MAP FOREACH KEYS VALUES LENGTH RANGE SLICE
-        SELECT TAKE DROP NTH LAST FLATTEN REDUCE + - * / MOD
-        == != > >= < <= OR AND NOT IN BOOL INT FLOAT
-        VARIABLES ! @ !@ INTERPRET EXPORT USE-MODULES REC
-      ]
-      words.concat(global_words)
+      new_stop -= 1 if eos_found && !done
+      new_stop += 1 if !eos_found && !done
 
-      # Collect from module stack
-      @execution_state.module_stack.each do |mod|
-        if mod.respond_to?(:words) && mod.words
-          words.concat(mod.words.map(&:name))
+      # Execute only tokens we have not executed previously
+      (@streaming_token_index...new_stop).each do |i|
+        token = tokens[i]
+        next unless token
+
+        handle_token(token)
+
+        if @stream && (token.type != TokenType::WORD || token.string != "START-LOG")
+          yield token.string
         end
+        @previous_token = token
       end
 
-      words.uniq.compact
-    end
-
-    # Calculate Levenshtein distance between two strings
-    # @param [String] str1
-    # @param [String] str2
-    # @return [Integer] The edit distance between the strings
-    def levenshtein_distance(str1, str2)
-      return str2.length if str1.empty?
-      return str1.length if str2.empty?
-
-      # Create matrix
-      matrix = Array.new(str1.length + 1) { Array.new(str2.length + 1, 0) }
-
-      # Initialize first row and column
-      (0..str1.length).each { |i| matrix[i][0] = i }
-      (0..str2.length).each { |j| matrix[0][j] = j }
-
-      # Fill matrix
-      (1..str1.length).each do |i|
-        (1..str2.length).each do |j|
-          cost = (str1[i - 1] == str2[j - 1]) ? 0 : 1
-          matrix[i][j] = [
-            matrix[i - 1][j] + 1,      # deletion
-            matrix[i][j - 1] + 1,      # insertion
-            matrix[i - 1][j - 1] + cost # substitution
-          ].min
-        end
+      # Done with this tokenizer
+      @tokenizer_stack.pop
+
+      if @stream && !eos_found && delta
+        # Yield string delta if we're streaming and tokenizer has a delta
+        new_portion = delta[@previous_delta_length..-1]
+
+        yield({ string_delta: new_portion }) if new_portion && !new_portion.empty?
+        @previous_delta_length = delta.length
+      end
+
+      if done
+        end_stream
+        return
       end
 
-      matrix[str1.length][str2.length]
+      # Update our pointer and reset if done
+      @streaming_token_index = new_stop
+    end
+
+    def start_stream
+      @stream = true
+      @previous_delta_length = 0
+      @streaming_token_index = 0
+    end
+
+    def end_stream
+      @stream = false
+      @previous_delta_length = 0
+      @streaming_token_index = 0
+    end
+
+    private
+
+    def find_last_word_or_eos(tokens)
+      tokens.rindex { |token| token.type == TokenType::WORD || token.type == TokenType::EOS } || -1
+    end
+
+    # Wrapper methods for literal handlers to match expected interface
+    def to_bool(str)
+      Forthic.to_bool(str)
+    end
+
+    def to_float(str)
+      Forthic.to_float(str)
+    end
+
+    def to_int(str)
+      Forthic.to_int(str)
+    end
+
+    def to_time(str)
+      Forthic.to_time(str)
+    end
+  end
+
+  # Duplicate an interpreter
+  #
+  # @param interp [Interpreter] The interpreter to duplicate
+  # @return [Interpreter] A new interpreter with copied state
+  def self.dup_interpreter(interp)
+    # Create new interpreter of the same type as the source
+    result_interp = interp.class.new([], interp.get_timezone)
+
+    # Use copy() instead of dup() to preserve module_prefixes
+    result_interp.instance_variable_set(:@app_module, interp.instance_variable_get(:@app_module).copy(result_interp))
+    result_interp.instance_variable_set(:@module_stack, [result_interp.instance_variable_get(:@app_module)])
+
+    # Use Stack.dup() method
+    result_interp.instance_variable_set(:@stack, interp.instance_variable_get(:@stack).dup)
+
+    # Share registered modules reference (modules are shared, not copied)
+    result_interp.instance_variable_set(:@registered_modules, interp.instance_variable_get(:@registered_modules))
+
+    # Copy error handler if present
+    handle_error = interp.instance_variable_get(:@handle_error)
+    result_interp.instance_variable_set(:@handle_error, handle_error) if handle_error
+
+    result_interp
+  end
+
+  # StandardInterpreter - Full-featured interpreter with standard library
+  #
+  # Extends Interpreter and automatically imports standard modules:
+  # - CoreModule: Stack operations, variables, module system, control flow
+  # - ArrayModule: Array/collection operations
+  # - RecordModule: Record/hash operations
+  # - StringModule: String operations
+  # - MathModule: Mathematical operations
+  # - BooleanModule: Boolean logic
+  # - JsonModule: JSON operations
+  # - DateTimeModule: DateTime operations
+  #
+  # For most use cases, use this class. Use Interpreter if you need
+  # full control over which modules are loaded.
+  class StandardInterpreter < Interpreter
+    def initialize(modules = [], timezone = "UTC")
+      # Don't pass modules to super - we'll import them after stdlib
+      super([], timezone)
+
+      # Import standard library modules FIRST (checked last during lookup)
+      # This allows user modules to shadow stdlib words
+      import_standard_library
+
+      # Import user modules AFTER stdlib (checked first during lookup)
+      import_modules(modules)
+    end
+
+    private
+
+    def import_standard_library
+      require_relative 'modules/standard/core_module'
+      require_relative 'modules/standard/array_module'
+      require_relative 'modules/standard/record_module'
+      require_relative 'modules/standard/string_module'
+      require_relative 'modules/standard/math_module'
+      require_relative 'modules/standard/boolean_module'
+      require_relative 'modules/standard/json_module'
+      require_relative 'modules/standard/datetime_module'
+
+      stdlib = [
+        Modules::CoreModule.new,
+        Modules::ArrayModule.new,
+        Modules::RecordModule.new,
+        Modules::StringModule.new,
+        Modules::MathModule.new,
+        Modules::BooleanModule.new,
+        Modules::JsonModule.new,
+        Modules::DateTimeModule.new,
+      ]
+
+      # Import unprefixed at the BOTTOM of module stack
+      # This ensures they're checked LAST during find_word()
+      stdlib.each do |mod|
+        import_module(mod, "")
+      end
     end
   end
 end