forthic 0.1.0 → 0.3.0

data/lib/forthic/interpreter.rb

@@ -1,154 +1,357 @@
 # frozen_string_literal: true
 
-require_relative 'forthic_error'
 require_relative 'tokenizer'
-require_relative 'token'
-require_relative 'code_location'
-require_relative 'positioned_string'
-require_relative 'words/word'
-require_relative 'words/push_value_word'
-require_relative 'words/start_module_word'
-require_relative 'words/end_module_word'
-require_relative 'words/end_array_word'
-require_relative 'words/definition_word'
-require_relative 'global_module'
+require_relative 'module'
+require_relative 'errors'
+require_relative 'literals'
+require 'time'
 
 module Forthic
-  class Interpreter
-    attr_accessor :stack, :global_module, :app_module, :module_stack, :registered_modules,
-                  :is_compiling, :should_stop, :is_memo_definition, :cur_definition,
-                  :screens, :default_module_flags, :module_flags, :string_location,
-                  :word_counts, :is_profiling, :start_profile_time, :timestamps
+  # Timestamp - Struct for profiling timestamps
+  Timestamp = Struct.new(:label, :time_ms, keyword_init: true)
+
+  # StartModuleWord - Handles module creation and switching
+  #
+  # Pushes a module onto the module stack, creating it if necessary.
+  # An empty name refers to the app module.
+  class StartModuleWord < Word
+    def execute(interp)
+      # The app module is the only module with a blank name
+      if @name == ""
+        interp.module_stack_push(interp.get_app_module)
+        return
+      end
+
+      # If the module is used by the current module, push it onto the stack, otherwise
+      # create a new module.
+      mod = interp.cur_module.find_module(@name)
+      unless mod
+        mod = Module.new(@name)
+        interp.cur_module.register_module(mod.name, mod.name, mod)
+
+        # If we're at the app module, also register with interpreter
+        if interp.cur_module.name == ""
+          interp.register_module(mod)
+        end
+      end
+      interp.module_stack_push(mod)
+    end
+  end
+
+  # EndModuleWord - Pops the current module from the module stack
+  #
+  # Completes module context and returns to the previous module.
+  class EndModuleWord < Word
+    def initialize
+      super("}")
+    end
+
+    def execute(interp)
+      interp.module_stack_pop
+    end
+  end
 
+  # EndArrayWord - Collects items from stack into an array
+  #
+  # Pops items from the stack until a START_ARRAY token is found,
+  # then pushes them as a single array in the correct order.
+  class EndArrayWord < Word
     def initialize
-      @stack = []
+      super("]")
+    end
+
+    def execute(interp)
+      items = []
+      item = interp.stack_pop
+      # NOTE: This won't infinite loop because interp.stack_pop will eventually fail
+      loop do
+        break if item.is_a?(Token) && item.type == TokenType::START_ARRAY
+        items << item
+        item = interp.stack_pop
+      end
+      items.reverse!
+      interp.stack_push(items)
+    end
+  end
+
+  # Stack - Wrapper for the interpreter's data stack
+  #
+  # Provides stack operations with support for array indexing.
+  # Handles PositionedString unwrapping and provides JSON serialization.
+  # Items can be accessed with bracket notation (e.g., stack[0]).
+  class Stack
+    def initialize(items = [])
+      @items = items
+    end
+
+    # Array-like access methods
+    def [](index)
+      @items[index]
+    end
+
+    def []=(index, value)
+      @items[index] = value
+    end
+
+    def length
+      @items.length
+    end
+
+    alias size length
+
+    def get_items
+      @items.map do |item|
+        if item.is_a?(PositionedString)
+          item.value
+        else
+          item
+        end
+      end
+    end
+
+    def get_raw_items
+      @items
+    end
+
+    def set_raw_items(items)
+      @items = items
+    end
+
+    def to_json(*args)
+      @items.to_json(*args)
+    end
+
+    def pop
+      @items.pop
+    end
+
+    def push(item)
+      @items.push(item)
+    end
+
+    # Duplicate stack with a shallow copy of items
+    def dup
+      Stack.new(@items.dup)
+    end
+  end
+
+  # Interpreter - Base Forthic interpreter
+  #
+  # Core interpreter that tokenizes and executes Forthic code.
+  # Manages the data stack, module stack, and execution context.
+  #
+  # Features:
+  # - Stack-based execution model
+  # - Module system with imports and namespacing
+  # - Literal handlers for parsing values (numbers, dates, booleans, etc.)
+  # - Error handling with recovery attempts
+  # - Profiling and performance tracking
+  # - Streaming execution support
+  #
+  # Note: This is the base interpreter without standard library modules.
+  # Use StandardInterpreter for a full-featured interpreter with stdlib.
+  class Interpreter
+    attr_reader :timezone, :stack
+    attr_accessor :handle_error, :max_attempts, :on_word_execute
+
+    def initialize(modules = [], timezone = "UTC")
+      @timezone = timezone
+      @stack = Stack.new
+
+      @tokenizer_stack = []
+      @max_attempts = 3
+      @handle_error = nil
+
+      @app_module = Module.new("")
+      @app_module.set_interp(self)
+      @module_stack = [@app_module]
       @registered_modules = {}
       @is_compiling = false
-      @should_stop = false
       @is_memo_definition = false
       @cur_definition = nil
-      @screens = {}
-      @default_module_flags = {}
-      @module_flags = {}
-      @string_location = nil
 
-      @global_module = GlobalModule.new(self)
-      @app_module = ForthicModule.new("", self)
-      @module_stack = [@app_module]
+      # Debug support
+      @string_location = nil
+      @previous_token = nil
 
+      # Profiling support
       @word_counts = {}
       @is_profiling = false
       @start_profile_time = nil
       @timestamps = []
+
+      # Streaming support
+      @streaming_token_index = 0
+      @stream = false
+      @previous_delta_length = 0
+
+      # Progress callback for streaming execution
+      @on_word_execute = nil
+      @word_execution_count = 0
+      @total_words_estimate = 0
+
+      # Literal handlers
+      @literal_handlers = []
+      register_standard_literals
+
+      # If modules are provided, import them unprefixed as a convenience
+      import_modules(modules)
+    end
+
+    def get_timezone
+      @timezone
     end
 
-    def halt
-      @should_stop = true
+    def set_timezone(timezone)
+      @timezone = timezone
     end
 
-    # @return [ForthicModule]
     def get_app_module
       @app_module
     end
 
-    # @return [CodeLocation, nil]
+    def get_top_input_string
+      return "" if @tokenizer_stack.empty?
+      @tokenizer_stack[0].get_input_string
+    end
+
+    def get_tokenizer
+      @tokenizer_stack.last
+    end
+
     def get_string_location
       @string_location
     end
 
-    # @param [String] module_id
-    # @param [Hash] flags
-    def set_flags(module_id, flags)
-      @default_module_flags[module_id] = flags
-      @module_flags[module_id] = flags
+    def set_max_attempts(max_attempts)
+      @max_attempts = max_attempts
     end
 
-    # @param [String] module_id
-    # @return [Hash]
-    def get_flags(module_id)
-      module_flags = @module_flags[module_id] || {}
-      result = module_flags.dup
-      @module_flags[module_id] = @default_module_flags[module_id].dup
-      result
+    def set_error_handler(handle_error)
+      @handle_error = handle_error
     end
 
-    # @param [String] module_id
-    # @param [Hash] flags
-    def modify_flags(module_id, flags)
-      module_flags = @module_flags[module_id] || {}
-      @module_flags[module_id] = module_flags.merge(flags)
+    def get_max_attempts
+      @max_attempts
+    end
+
+    def get_error_handler
+      @handle_error
     end
 
     def reset
-      @stack = []
+      @stack = Stack.new
       @app_module.variables = {}
+
       @module_stack = [@app_module]
       @is_compiling = false
       @is_memo_definition = false
       @cur_definition = nil
+
+      # Debug support
       @string_location = nil
     end
 
-    # @param [String] screen_name
-    # @return [String]
-    def get_screen_forthic(screen_name)
-      screen = @screens[screen_name]
-      raise ForthicError.new("interpreter-199", "Unable to find screen \"#{screen_name}\"", "Hmmm...something went wrong. Please file a ticket if this continues to happen") unless screen
-      screen
+    def run(string, reference_location = nil)
+      @tokenizer_stack.push(Tokenizer.new(string, reference_location))
+
+      if @handle_error
+        execute_with_recovery
+      else
+        run_with_tokenizer(@tokenizer_stack.last)
+      end
+
+      @tokenizer_stack.pop
+      true
+    end
+
+    def execute_with_recovery(num_attempts = 0)
+      num_attempts += 1
+      if num_attempts > @max_attempts
+        raise TooManyAttemptsError.new(
+          get_top_input_string,
+          num_attempts,
+          @max_attempts
+        )
+      end
+      continue_execution
+      num_attempts
+    rescue => e
+      raise unless @handle_error
+      @handle_error.call(e, self)
+      execute_with_recovery(num_attempts)
     end
 
-    # @param [String] string
-    # @param [CodeLocation, nil] reference_location
-    # @return [Boolean]
-    def run(string, reference_location = nil)
-      tokenizer = Tokenizer.new(string, reference_location)
-      run_with_tokenizer(tokenizer)
+    def continue_execution
+      run_with_tokenizer(@tokenizer_stack.last)
     end
 
-    # @param [Tokenizer] tokenizer
-    # @return [Boolean]
     def run_with_tokenizer(tokenizer)
       token = nil
       loop do
+        @previous_token = token
         token = tokenizer.next_token
         handle_token(token)
-        break if token.type == TokenType::EOS || @should_stop
-        next if [TokenType::START_DEF, TokenType::END_DEF, TokenType::COMMENT].include?(token.type) || @is_compiling
+        break if token.type == TokenType::EOS
       end
-      true
-    # rescue => e
-    #   error = ForthicError.new("interpreter-213", "Ran into an error executing this '#{token.string}'", "If there is an unknown error in the stack details, please file a ticket so we can resolve it.", token.location)
-    #   error.set_caught_error(e)
-    #   raise error
+      true # Done executing
     end
 
-    # @return [ForthicModule]
     def cur_module
       @module_stack.last
     end
 
-    # @param [String] name
-    # @return [ForthicModule]
     def find_module(name)
       result = @registered_modules[name]
-      raise ForthicError.new("interpreter-236", "Couldn't find '#{name}' module", "This is most likely a typo in your Forthic code. Please check to see if '#{name}' is properly spelled and that you have permission to access it") unless result
+      unless result
+        raise UnknownModuleError.new(
+          get_top_input_string,
+          name,
+          location: @string_location
+        )
+      end
+      result
+    end
+
+    def stack_peek
+      top = @stack[@stack.length - 1]
+      result = top
+      result = top.value if top.is_a?(PositionedString)
       result
     end
 
-    # @param [Object] val
     def stack_push(val)
       @stack.push(val)
     end
 
-    # @return [Object]
     def stack_pop
-      raise ForthicError.new("interpreter-251", "Stack underflow", "This happens when we expect something to be on the stack, but it's empty. This is caused by a logical error in the Forthic and can be resolved through debugging.") if @stack.empty?
+      if @stack.length.zero?
+        tokenizer = @tokenizer_stack.length > 0 ? get_tokenizer : nil
+        raise StackUnderflowError.new(
+          get_top_input_string,
+          location: tokenizer&.get_token_location
+        )
+      end
       result = @stack.pop
-      @string_location = result.is_a?(PositionedString) ? result.location : nil
-      result.is_a?(PositionedString) ? result.value_of : result
+
+      # If we have a PositionedString, we need to record the location
+      @string_location = nil
+      if result.is_a?(PositionedString)
+        positioned_string = result
+        result = positioned_string.value
+        @string_location = positioned_string.location
+      end
+      result
+    end
+
+    def get_stack
+      @stack
+    end
+
+    def set_stack(stack)
+      @stack = stack
     end
 
-    # @param [ForthicModule] mod
     def module_stack_push(mod)
       @module_stack.push(mod)
     end
@@ -157,47 +360,147 @@ module Forthic
       @module_stack.pop
     end
 
-    # @param [ForthicModule] mod
     def register_module(mod)
       @registered_modules[mod.name] = mod
+      mod.set_interp(self)
+    end
+
+    # If names is an array of strings, import each module without a prefix (empty string)
+    # If names is an array of arrays, import each module using the first element as the
+    # module name and the second element as the prefix
+    def use_modules(names)
+      names.each do |name|
+        module_name = name
+        prefix = "" # Default to empty prefix (no prefix)
+        if name.is_a?(Array)
+          module_name = name[0]
+          prefix = name[1] # Allow explicit prefix specification
+        end
+        mod = find_module(module_name)
+        get_app_module.import_module(prefix, mod, self)
+      end
+    end
+
+    # A convenience method to register and use a module
+    def import_module(mod, prefix = "")
+      register_module(mod)
+      use_modules([[mod.name, prefix]])
+    end
+
+    def import_modules(modules)
+      modules.each do |mod|
+        import_module(mod)
+      end
+    end
+
+    # Transforms simple module names to unprefixed imports: "math" -> ["math", ""]
+    # Preserves explicit prefix specifications: ["math", "m"] -> ["math", "m"]
+    def use_modules_unprefixed(names)
+      unprefixed = names.map do |name|
+        name.is_a?(Array) ? name : [name, ""]
+      end
+      use_modules(unprefixed)
     end
 
-    # @param [ForthicModule] mod
     def run_module_code(mod)
       module_stack_push(mod)
-      run(mod.forthic_code)
+      begin
+        # Set source to module name when running module code
+        module_location = CodeLocation.new(source: mod.name)
+        run(mod.forthic_code, module_location)
+      rescue => e
+        raise ModuleError.new(
+          get_top_input_string,
+          mod.name,
+          e,
+          location: @string_location,
+          cause: e
+        )
+      end
+
       module_stack_pop
-    rescue => e
-      error = ForthicError.new("interpreter-278", "Something went wrong when running the module #{mod.name}", "TODO: File a ticket")
-      error.set_caught_error(e)
-      raise error
     end
 
-    # @param [String] name
-    # @return [Word, nil]
+    # ======================
+    # Literal Handlers
+
+    # Register standard literal handlers
+    # Order matters: more specific handlers first
+    def register_standard_literals
+      @literal_handlers = [
+        method(:to_bool).to_proc,                                  # TRUE, FALSE
+        method(:to_float).to_proc,                                 # 3.14
+        Forthic.to_zoned_datetime(@timezone),                      # 2020-06-05T10:15:00Z
+        Forthic.to_literal_date(@timezone),                        # 2020-06-05, YYYY-MM-DD
+        method(:to_time).to_proc,                                  # 9:00, 11:30 PM
+        method(:to_int).to_proc,                                   # 42
+      ]
+    end
+
+    # Register a custom literal handler
+    # Handlers are checked in registration order
+    def register_literal_handler(handler)
+      @literal_handlers << handler
+    end
+
+    # Unregister a literal handler
+    def unregister_literal_handler(handler)
+      @literal_handlers.delete(handler)
+    end
+
+    # Try to parse string as a literal value
+    # Returns PushValueWord if successful, nil otherwise
+    def find_literal_word(name)
+      @literal_handlers.each do |handler|
+        value = handler.call(name)
+        return PushValueWord.new(name, value) unless value.nil?
+      end
+      nil
+    end
+
+    # ======================
+    # Find Word
+
     def find_word(name)
+      # 1. Check module stack (dictionary words + variables)
       result = nil
-      @module_stack.reverse_each do |m|
+      (@module_stack.length - 1).downto(0) do |i|
+        m = @module_stack[i]
         result = m.find_word(name)
         break if result
       end
-      result ||= @global_module.find_word(name)
+
+      # 2. Check literal handlers as fallback
+      result = find_literal_word(name) unless result
+
+      # 3. Throw error if still not found
+      unless result
+        raise UnknownWordError.new(
+          get_top_input_string,
+          name,
+          location: get_string_location
+        )
+      end
+
       result
     end
 
+    # ======================
+    # Profiling
+
     def start_profiling
       @is_profiling = true
       @timestamps = []
-      @start_profile_time = Time.now
+      @start_profile_time = Time.now.to_f * 1000 # milliseconds
       add_timestamp("START")
       @word_counts = {}
     end
 
-    # @param [Word] word
     def count_word(word)
       return unless @is_profiling
-      @word_counts[word.name] ||= 0
-      @word_counts[word.name] += 1
+      name = word.name
+      @word_counts[name] ||= 0
+      @word_counts[name] += 1
     end
 
     def stop_profiling
@@ -205,99 +508,141 @@ module Forthic
       @is_profiling = false
     end
 
-    # @param [String] label
     def add_timestamp(label)
       return unless @is_profiling
-      timestamp = { label: label, time_ms: (Time.now - @start_profile_time) * 1000 }
-      @timestamps.push(timestamp)
+      timestamp = Timestamp.new(
+        label: label,
+        time_ms: (Time.now.to_f * 1000) - @start_profile_time
+      )
+      @timestamps << timestamp
     end
 
-    # @return [Array<Hash>]
     def word_histogram
-      @word_counts.map { |name, count| { word: name, count: count } }.sort_by { |item| -item[:count] }
+      items = []
+      @word_counts.each do |name, count|
+        items << { word: name, count: count }
+      end
+      items.sort_by { |item| -item[:count] }
     end
 
-    # @return [Array<Hash>]
     def profile_timestamps
       @timestamps
     end
 
-    # @param [Token] token
+    # ======================
+    # Handle tokens
+
     def handle_token(token)
       case token.type
-      when TokenType::STRING then handle_string_token(token)
-      when TokenType::COMMENT then handle_comment_token(token)
-      when TokenType::START_ARRAY then handle_start_array_token(token)
-      when TokenType::END_ARRAY then handle_end_array_token(token)
-      when TokenType::START_MODULE then handle_start_module_token(token)
-      when TokenType::END_MODULE then handle_end_module_token(token)
-      when TokenType::START_DEF then handle_start_definition_token(token)
-      when TokenType::START_MEMO then handle_start_memo_token(token)
-      when TokenType::END_DEF then handle_end_definition_token(token)
-      when TokenType::WORD then handle_word_token(token)
-      when TokenType::EOS then return
+      when TokenType::STRING
+        handle_string_token(token)
+      when TokenType::COMMENT
+        handle_comment_token(token)
+      when TokenType::START_ARRAY
+        handle_start_array_token(token)
+      when TokenType::END_ARRAY
+        handle_end_array_token(token)
+      when TokenType::START_MODULE
+        handle_start_module_token(token)
+      when TokenType::END_MODULE
+        handle_end_module_token(token)
+      when TokenType::START_DEF
+        handle_start_definition_token(token)
+      when TokenType::START_MEMO
+        handle_start_memo_token(token)
+      when TokenType::END_DEF
+        handle_end_definition_token(token)
+      when TokenType::DOT_SYMBOL
+        handle_dot_symbol_token(token)
+      when TokenType::WORD
+        handle_word_token(token)
+      when TokenType::EOS
+        if @is_compiling
+          raise MissingSemicolonError.new(
+            get_top_input_string,
+            location: @previous_token&.location
+          )
+        end
       else
-        raise ForthicError.new("interpreter-362", "Hmmm...the interpreter doesn't know what to make of '#{token.string}'", "This is most likely caused by a typo in the Forthic code and can be resolved by debugging.", token.location)
+        raise UnknownTokenError.new(
+          get_top_input_string,
+          token.string,
+          location: @string_location
+        )
       end
     end
 
-    # @param [Token] token
     def handle_string_token(token)
       value = PositionedString.new(token.string, token.location)
       handle_word(PushValueWord.new("<string>", value))
     end
 
-    # @param [Token] token
+    def handle_dot_symbol_token(token)
+      value = PositionedString.new(token.string, token.location)
+      handle_word(PushValueWord.new("<dot-symbol>", value))
+    end
+
+    # Start/end module tokens are treated as IMMEDIATE words *and* are also compiled
     def handle_start_module_token(token)
       word = StartModuleWord.new(token.string)
+
       @cur_definition.add_word(word) if @is_compiling
-      count_word(word)
+      count_word(word) # For profiling
       word.execute(self)
     end
 
-    # @param [Token] _token
     def handle_end_module_token(_token)
       word = EndModuleWord.new
+
       @cur_definition.add_word(word) if @is_compiling
       count_word(word)
       word.execute(self)
     end
 
-    # @param [Token] token
     def handle_start_array_token(token)
       handle_word(PushValueWord.new("<start_array_token>", token))
     end
 
-    # @param [Token] _token
     def handle_end_array_token(_token)
       handle_word(EndArrayWord.new)
     end
 
-    # @param [Token] _token
     def handle_comment_token(_token)
-      # Handle comment token (no-op)
+      # Comment handling (currently no-op)
     end
 
-    # @param [Token] token
     def handle_start_definition_token(token)
-      raise ForthicError.new("interpreter-407", "A definition was started while an existing definition was not ended", "This is probably caused by a missing semicolon. To resolve, ensure that all word definitions end with semicolons.", token.location) if @is_compiling
+      if @is_compiling
+        raise MissingSemicolonError.new(
+          get_top_input_string,
+          location: @previous_token&.location
+        )
+      end
       @cur_definition = DefinitionWord.new(token.string)
       @is_compiling = true
       @is_memo_definition = false
     end
 
-    # @param [Token] token
     def handle_start_memo_token(token)
-      raise ForthicError.new("interpreter-420", "A memo definition was started while an existing definition was not ended", "This is probably caused by a missing semicolon. To resolve, ensure that all word definitions end with semicolons.", token.location) if @is_compiling
+      if @is_compiling
+        raise MissingSemicolonError.new(
+          get_top_input_string,
+          location: @previous_token&.location
+        )
+      end
       @cur_definition = DefinitionWord.new(token.string)
       @is_compiling = true
       @is_memo_definition = true
     end
 
-    # @param [Token] token
     def handle_end_definition_token(token)
-      raise ForthicError.new("interpreter-433", "A definition was ended when one hadn't been started yet", "This is probably caused by an extra semicolon. To resolve, ensure that there are no spurious semicolons in the Forthic code.", token.location) unless @is_compiling
-      raise ForthicError.new("interpreter-440", "Cannot finish definition because there is no current definition", "Please file a ticket", token.location) unless @cur_definition
+      unless @is_compiling && @cur_definition
+        raise ExtraSemicolonError.new(
+          get_top_input_string,
+          location: token.location
+        )
+      end
+
       if @is_memo_definition
         cur_module.add_memo_words(@cur_definition)
       else
@@ -306,23 +651,227 @@ module Forthic
       @is_compiling = false
     end
 
-    # @param [Token] token
     def handle_word_token(token)
-      word = find_word(token.string)
-      raise ForthicError.new("interpreter-458", "Could not find word: #{token.string}", "Check to see if you have a typo in your word or the definition of that word", token.location) unless word
+      word = find_word(token.string) # Throws UnknownWordError if not found
       handle_word(word, token.location)
     end
 
-    # @param [Word] word
-    # @param [CodeLocation, nil] location
     def handle_word(word, location = nil)
       if @is_compiling
         word.set_location(location)
         @cur_definition.add_word(word)
       else
         count_word(word)
+
+        # Notify progress callback if set (for streaming execution)
+        if @on_word_execute && word.is_a?(DefinitionWord)
+          @word_execution_count += 1
+          @on_word_execute.call(word.name, @word_execution_count, @total_words_estimate)
+        end
+
         word.execute(self)
       end
     end
+
+    # Estimate the number of words in code (for progress tracking)
+    def estimate_word_count(code)
+      # Simple heuristic: count uppercase words (likely Forthic words)
+      code.scan(/[A-Z][A-Z0-9_-]*/).length
+    end
+
+    # Set up progress tracking for streaming execution
+    def setup_progress_tracking(code)
+      @word_execution_count = 0
+      @total_words_estimate = estimate_word_count(code)
+    end
+
+    # Reset progress tracking
+    def reset_progress_tracking
+      @word_execution_count = 0
+      @total_words_estimate = 0
+      @on_word_execute = nil
+    end
+
+    # Streaming execution support
+    def streaming_run(code_stream, done, reference_location = nil)
+      # Create a new Tokenizer for the full string
+      tokenizer = Tokenizer.new(code_stream, reference_location, done ? false : true)
+      tokens = []
+      eos_found = false
+
+      @tokenizer_stack.push(tokenizer)
+
+      # Gather tokens from the beginning
+      loop do
+        token = tokenizer.next_token
+        break unless token
+
+        # If we hit an EOS token then push it and break
+        if token.type == TokenType::EOS
+          tokens << token
+          eos_found = true
+          break
+        end
+
+        tokens << token
+      end
+
+      delta = eos_found ? nil : tokenizer.get_string_delta
+
+      new_stop = find_last_word_or_eos(tokens)
+
+      new_stop -= 1 if eos_found && !done
+      new_stop += 1 if !eos_found && !done
+
+      # Execute only tokens we have not executed previously
+      (@streaming_token_index...new_stop).each do |i|
+        token = tokens[i]
+        next unless token
+
+        handle_token(token)
+
+        if @stream && (token.type != TokenType::WORD || token.string != "START-LOG")
+          yield token.string
+        end
+        @previous_token = token
+      end
+
+      # Done with this tokenizer
+      @tokenizer_stack.pop
+
+      if @stream && !eos_found && delta
+        # Yield string delta if we're streaming and tokenizer has a delta
+        new_portion = delta[@previous_delta_length..-1]
+
+        yield({ string_delta: new_portion }) if new_portion && !new_portion.empty?
+        @previous_delta_length = delta.length
+      end
+
+      if done
+        end_stream
+        return
+      end
+
+      # Update our pointer and reset if done
+      @streaming_token_index = new_stop
+    end
+
+    def start_stream
+      @stream = true
+      @previous_delta_length = 0
+      @streaming_token_index = 0
+    end
+
+    def end_stream
+      @stream = false
+      @previous_delta_length = 0
+      @streaming_token_index = 0
+    end
+
+    private
+
+    def find_last_word_or_eos(tokens)
+      tokens.rindex { |token| token.type == TokenType::WORD || token.type == TokenType::EOS } || -1
+    end
+
+    # Wrapper methods for literal handlers to match expected interface
+    def to_bool(str)
+      Forthic.to_bool(str)
+    end
+
+    def to_float(str)
+      Forthic.to_float(str)
+    end
+
+    def to_int(str)
+      Forthic.to_int(str)
+    end
+
+    def to_time(str)
+      Forthic.to_time(str)
+    end
+  end
+
+  # Duplicate an interpreter
+  #
+  # @param interp [Interpreter] The interpreter to duplicate
+  # @return [Interpreter] A new interpreter with copied state
+  def self.dup_interpreter(interp)
+    # Create new interpreter of the same type as the source
+    result_interp = interp.class.new([], interp.get_timezone)
+
+    # Use copy() instead of dup() to preserve module_prefixes
+    result_interp.instance_variable_set(:@app_module, interp.instance_variable_get(:@app_module).copy(result_interp))
+    result_interp.instance_variable_set(:@module_stack, [result_interp.instance_variable_get(:@app_module)])
+
+    # Use Stack.dup() method
+    result_interp.instance_variable_set(:@stack, interp.instance_variable_get(:@stack).dup)
+
+    # Share registered modules reference (modules are shared, not copied)
+    result_interp.instance_variable_set(:@registered_modules, interp.instance_variable_get(:@registered_modules))
+
+    # Copy error handler if present
+    handle_error = interp.instance_variable_get(:@handle_error)
+    result_interp.instance_variable_set(:@handle_error, handle_error) if handle_error
+
+    result_interp
+  end
+
+  # StandardInterpreter - Full-featured interpreter with standard library
+  #
+  # Extends Interpreter and automatically imports standard modules:
+  # - CoreModule: Stack operations, variables, module system, control flow
+  # - ArrayModule: Array/collection operations
+  # - RecordModule: Record/hash operations
+  # - StringModule: String operations
+  # - MathModule: Mathematical operations
+  # - BooleanModule: Boolean logic
+  # - JsonModule: JSON operations
+  # - DateTimeModule: DateTime operations
+  #
+  # For most use cases, use this class. Use Interpreter if you need
+  # full control over which modules are loaded.
+  class StandardInterpreter < Interpreter
+    def initialize(modules = [], timezone = "UTC")
+      # Don't pass modules to super - we'll import them after stdlib
+      super([], timezone)
+
+      # Import standard library modules FIRST (checked last during lookup)
+      # This allows user modules to shadow stdlib words
+      import_standard_library
+
+      # Import user modules AFTER stdlib (checked first during lookup)
+      import_modules(modules)
+    end
+
+    private
+
+    def import_standard_library
+      require_relative 'modules/standard/core_module'
+      require_relative 'modules/standard/array_module'
+      require_relative 'modules/standard/record_module'
+      require_relative 'modules/standard/string_module'
+      require_relative 'modules/standard/math_module'
+      require_relative 'modules/standard/boolean_module'
+      require_relative 'modules/standard/json_module'
+      require_relative 'modules/standard/datetime_module'
+
+      stdlib = [
+        Modules::CoreModule.new,
+        Modules::ArrayModule.new,
+        Modules::RecordModule.new,
+        Modules::StringModule.new,
+        Modules::MathModule.new,
+        Modules::BooleanModule.new,
+        Modules::JsonModule.new,
+        Modules::DateTimeModule.new,
+      ]
+
+      # Import unprefixed at the BOTTOM of module stack
+      # This ensures they're checked LAST during find_word()
+      stdlib.each do |mod|
+        import_module(mod, "")
+      end
+    end
   end
-end
+end