kanocc 0.1.0

data/lib/kanocc/scanner.rb

@@ -0,0 +1,165 @@
+#  
+#  Copyright 2008 Christian Surlykke
+#
+#  This file is part of Kanocc.
+#require 'logger'
+
+#  Kanocc is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License, version 3 
+#  as published by the Free Software Foundation.
+#
+#  Kanocc is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License, version 3 for more details.
+#
+#  You should have received a copy of the GNU General Public License,
+#  version 3 along with Kanocc.  If not, see <http://www.gnu.org/licenses/>.
+#
+require 'stringio'
+require 'strscan'
+require "logger"
+module Kanocc
+  class Scanner
+    attr_accessor :logger
+    def initialize(init = {})
+      if init[:logger]
+        @logger = init[:logger] 
+      else
+        @logger = Logger.new(STDOUT)
+        @logger.level = Logger::WARN
+      end
+      @ws_regs = [/\s/]
+      @recognizables = []
+      @regexps = []
+    end
+    
+    def set_whitespace(*ws_regs)
+      @ws_regs = []
+      ws_regs.each do |ws_reg| 
+        unless ws_reg.is_a?(Regexp)
+          raise "set_whitespace must be given a list of Regexp's" 
+        end
+        @ws_regs << ws_reg
+      end
+    end
+    
+    def set_recognized(*rec)
+      @recognizables = []
+      rec.each do |r| 
+        if r.class == Class and r.ancestors.include?(Token)
+	  @recognizables = @recognizables + r.patterns
+        elsif r.is_a? String
+          @recognizables << {:literal => r,
+	                     :regexp  => Regexp.new(Regexp.escape(r))}
+        else
+          raise "set_recognized must be given a list of Tokens classes and or strings"
+        end
+      end
+    end
+    
+    def each_token(input)
+      if input.is_a?(IO) 
+        @input = input.readlines.join("")
+      elsif input.is_a?(String) 
+        @input = input
+      else
+        raise "Input must be a string or an IO object"
+      end 
+      @stringScanner = StringScanner.new(@input)
+      while match = do_match do
+        if match[:matches] 
+          @logger.debug("Yielding #{match}")
+          yield(match)
+        end
+        @stringScanner.pos += match[:length]
+      end
+    end
+
+    private
+    
+    def do_match
+      if @stringScanner.pos >= @stringScanner.string.length
+        return nil;
+      end
+      
+      token_match = match_token
+      whitespace_match = match_whitespace
+      
+      if whitespace_match[:length] > token_match[:length]
+        return whitespace_match
+      elsif token_match[:length] > 0
+        return token_match
+      else 
+	# So we've not been able to match tokens nor whitespace.
+        # We return the first character of the remaining input as a string
+        # literal
+        string = @stringScanner.string.slice(@stringScanner.pos, 1)
+        matches = [{:literal => string, 
+	            :regexp  => Regexp.new(Regexp.escape(string))}] 
+	return {:matches => matches,
+	        :string => string,
+	        :start_pos => @stringScanner.pos,
+		:length => 1}
+      end
+    end
+
+    def match_token
+      matches = []
+      max_length = 0 
+      @recognizables.each do |rec| 
+	if (len = @stringScanner.match?(rec[:regexp])) and len > 0 
+	  if len > max_length
+            # Now, we have a match longer than whatever we had, 
+            # so we discharge what we had, and save the new one
+            matches = [rec]
+            max_length = len
+          elsif len == max_length
+            # This regular expression matches a string of same length 
+            # as our previous match, so we prepare to return both
+            matches << rec 
+          end
+        end
+      end
+      start_pos = @stringScanner.pos
+      string = @stringScanner.string.slice(start_pos, max_length)
+      return {:matches => matches, 
+	      :string  => string, 
+	      :start_pos => start_pos, 
+	      :length => max_length}
+    end
+    
+    def match_whitespace
+      max_length = 0
+      for i in 0..@ws_regs.size - 1 do
+        len = @stringScanner.match?(@ws_regs[i]) || 0
+        if len > max_length
+          max_length = len
+        end
+      end
+      string = @stringScanner.string.slice(@stringScanner.pos, max_length)        
+      result = {:string => string, 
+	        :start_pos => @stringScanner.pos, 
+	        :length => max_length}
+      return result
+    end
+  end
+  
+end
+
+
+############################################
+#                Testing
+#require 'Token'
+#
+#class Number < Token
+#  set_pattern(/\d+/)
+#end
+#
+#scanner = KanoccScanner.new
+#scanner.set_recognized(Number, "Exit")
+#scanner.set_whitespace(/[ \t]/)
+#
+#scanner.eachTokenDo{|token|  print token.inspect, "\n"}
+
+

data/lib/kanocc/token.rb

@@ -0,0 +1,58 @@
+#  
+#  Copyright 2008 Christian Surlykke
+#
+#  This file is part of Kanocc.
+#
+#  Kanocc is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License, version 3 
+#  as published by the Free Software Foundation.
+#
+#  Kanocc is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License, version 3 for more details.
+#
+#  You should have received a copy of the GNU General Public License,
+#  version 3 along with Kanocc.  If not, see <http://www.gnu.org/licenses/>.
+#
+module Kanocc
+  class Token
+    attr_accessor :m
+    
+    @@patterns = Hash.new
+  
+    def ===(klass)
+      self.class == klass
+    end
+    
+    def Token.pattern(reg, &block)
+      raise "pattern must be given a Regexp as it's first argument" unless reg.is_a?(Regexp)
+      @@patterns[self] = [] unless @@patterns[self]
+      if block_given?
+        method_name = ("pattern " + reg.inspect).to_sym
+        define_method(method_name, &block)
+      else
+	method_name = nil
+      end
+      @@patterns[self] << {:token => self, 
+	                   :regexp => reg, 
+			   :method_name=>method_name}
+    end
+  
+    def Token.patterns
+      return @@patterns[self] || []
+    end
+
+    def is_a_kanocc_token?
+      return true
+    end
+
+    def Token.is_a_kanocc_grammarsymbol?
+      return true
+    end
+
+    def inspect
+      self.class.name 
+    end
+  end
+end

data/lib/todo

@@ -0,0 +1,3 @@
+Better handling of blocks
+LR Parsers
+Scanner.eachToken method

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+name: kanocc
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Christian Surlykke
+autorequire: kanocc
+bindir: bin
+cert_chain: []
+
+date: 2008-05-19 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: 
+email: ""
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- COPYING
+- lib/kanocc
+- lib/kanocc.rb
+- lib/todo
+- lib/kanocc/earley.rb
+- lib/kanocc/scanner.rb
+- lib/kanocc/grammar_rule.rb
+- lib/kanocc/nonterminal.rb
+- lib/kanocc/token.rb
+- examples/calculator.rb
+- examples/ruby_quiz_78.rb
+has_rdoc: false
+homepage: ""
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 0.9.5
+signing_key: 
+specification_version: 2
+summary: Kanocc - Kanocc ain't no compiler-compiler. A framework for syntax directed translation
+test_files: []
+