udon 0.0.0 → 0.0.1

data/Gemfile

@@ -6,10 +6,11 @@ source "http://rubygems.org"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "shoulda", ">= 0"
+  gem "minitest", ">= 0"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.4"
   gem "rcov", ">= 0"
   gem "reek", "~> 1.2.8"
   gem "roodi", "~> 2.1.0"
+  gem "genmachine", "~> 0.1.6"
 end

data/Rakefile

@@ -1,5 +1,6 @@
 # encoding: utf-8
 
+require 'rdoc/task'
 require 'rubygems'
 require 'bundler'
 begin
@@ -11,6 +12,16 @@ rescue Bundler::BundlerError => e
 end
 require 'rake'
 
+file "lib/udon/udon_parser.rb" => ["machines/udon.machine"] do |t|
+  sh "genmachine -o lib/udon/ -l ruby --no-executable -c UdonParser #{t.prerequisites.join(' ')}"
+end
+task :install => [:build_parser]
+
+task :build_parser do |t|
+  sh "rm -f lib/udon/udon_parser.rb"
+  Rake::Task['lib/udon/udon_parser.rb'].execute
+end
+
 require 'jeweler'
 Jeweler::Tasks.new do |gem|
   # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
@@ -29,14 +40,16 @@ require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+  test.verbose = false
 end
 
+task :test => ['lib/udon/udon_parser.rb']
+
 require 'rcov/rcovtask'
 Rcov::RcovTask.new do |test|
   test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+  test.verbose = false
   test.rcov_opts << '--exclude "gems/*"'
 end
 
@@ -55,7 +68,6 @@ end
 
 task :default => :test
 
-require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
 

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.1

data/lib/VERSION

@@ -0,0 +1 @@
+0.0.1

data/lib/udon/udon_parser.rb

@@ -0,0 +1,205 @@
+require 'strscan'
+$KCODE="U"
+
+module UdonParser
+  def self.parse(str) Parser.new(str).parse end
+  def self.parse_file(fname) Parser.new(IO.read(fname)).parse end
+
+  class UHash < Hash
+    def <<(kv) k,v = kv; self[k] = v end
+  end
+
+  class UString < String
+    def <<(v)
+      begin
+        super([v].pack('U*'))
+      rescue
+        super(v)
+      end
+    end
+  end
+
+  class UNode
+    attr_accessor :name, :m,:a,:c
+    def initialize(params={})
+      @m = params.delete(:m) || UHash.new
+      @m[:sline] ||= params.delete(:sline)
+      @m[:schr] ||= params.delete(:schr)
+      @a= params.delete(:a) || UHash.new
+      @c= params.delete(:c) || []
+      @name = params.delete(:name)
+    end
+    def <<(val) @c<<val end
+    def [](key) @c[key] end
+  end
+
+  class Parser < StringScanner
+    def init(source, opts={})
+      opts ||= {}
+      super ensure_encoding(source)
+      @global = {}
+    end
+
+    def ensure_encoding(source)
+      if defined?(::Encoding)
+        if source.encoding == ::Encoding::ASCII_8BIT
+          b = source[0, 4].bytes.to_a
+          source =
+            case
+            when b.size>=4 && b[0]==0 && b[1]==0 && b[2]==0
+              source.dup.force_encoding(::Encoding::UTF_32BE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[0]==0 && b[2]==0
+              source.dup.force_encoding(::Encoding::UTF_16BE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[1]==0 && b[2]==0 && b[3]==0
+              source.dup.force_encoding(::Encoding::UTF_32LE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[1]==0 && b[3]==0
+              source.dup.force_encoding(::Encoding::UTF_16LE).encode!(::Encoding::UTF_8)
+            else source.dup end
+        else source = source.encode(::Encoding::UTF_8) end
+        source.force_encoding(::Encoding::ASCII_8BIT)
+      else
+        b = source
+        source =
+          case
+          when b.size >= 4 && b[0] == 0 && b[1] == 0 && b[2] == 0; Iconv.iconv('utf-8', 'utf-32be', b)
+          when b.size >= 4 && b[0] == 0 && b[2] == 0; Iconv.iconv('utf-8', 'utf-16be', b)
+          when b.size >= 4 && b[1] == 0 && b[2] == 0 && b[3] == 0; Iconv.iconv('utf-8', 'utf-32le', b)
+          when b.size >= 4 && b[1] == 0 && b[3] == 0; Iconv.iconv('utf-8', 'utf-16le', b)
+          else b end
+      end
+      return source
+    end
+
+    def parse
+      reset
+      @line = 1
+      @pos = 1
+      @leading = true
+      @indent = 0
+      @ast = document
+      return @ast
+    end
+
+    private
+
+    def error(msg)
+      $stderr.puts "#{msg} | line: #{@line} | char: #{@pos}"
+    end
+
+    def global_state(c)
+      # Unicode newline characters & combinations
+      # Plus leading space for indents.
+      # Also tracks line and position for the AST
+      @last_is_newline = @last_is_space = false
+      case c
+      when 0x0b, 0x0c, 0x85, 0x2028, 0x2029
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x0a
+        nc = peek(4).unpack('U')[0]
+        if nc == 0x0d then getch; c = "\n\r" end
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x0d
+        nc = peek(4).unpack('U')[0]
+        if nc == 0x0a then getch; c = "\r\n" end
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x20
+        @indent += 1 if @leading
+        @last_is_space = true; @pos += 1
+      else @leading = false; @pos += 1 end
+      return @last_c = c
+    end
+
+    def nl?() return @last_is_newline end
+    def space?() return @last_is_space end
+
+    def nextchar
+      if @fwd then @fwd = false; return @last_c
+      else
+        c = getch
+        if c.nil?
+          c = :eof
+          @last_is_space = @last_is_newline = false
+          return @last_c = c
+        end
+        return global_state(c.unpack('U')[0])
+      end
+    end
+
+    def eof?() return @last_c == :eof end
+
+    def document(p=nil,name='document')
+      state=':data_or_child'
+      s = []
+      a ||= UString.new
+      b ||= UString.new
+      trash ||= UString.new
+      loop do
+        c = nextchar
+        state = '{eof}' if c==:eof
+        case state
+        when ':data_or_child'
+            case
+            when nl?; @fwd=true; (a<<b if b.size>0); b=UString.new; state=':data'; next
+            when space?; b<<c; next
+            when c==35,c==124; @fwd=true; (trash<<b if b.size>0); b=UString.new; state=':child'; next
+            else @fwd=true; (a<<b if b.size>0); b=UString.new; state=':data'; next
+            end
+        when ':child'
+            if c==35
+              @fwd=true; (s<<a if a.size>0); a=UString.new; state=comment(':data_or_child',s); next
+            end
+        when '{eof}'
+            @fwd=true; (a<<b if b.size>0); b=UString.new; (s<<a if a.size>0); a=UString.new; return(s)
+        when ':data'
+            case
+            when nl?; a<<c; (s<<a if a.size>0); a=UString.new; state=':data_or_child'; next
+            else a<<c; next
+            end
+        end
+      end
+    end
+
+    def comment(ns,p=nil,name='comment')
+      ipar=@indent
+      ibase=ipar+100
+      state=':1st:ws'
+      s = UNode.new(:name=>name,:sline=>@line,:schr=>@pos)
+      a ||= UString.new
+      loop do
+        c = nextchar
+        state = '{eof}' if c==:eof
+        case state
+        when ':nl'
+            case
+            when (@indent>ibase); @fwd=true; state=':child'; next
+            when nl?,(c>8&&c<11),space?; next
+            when (@indent<=ipar); @fwd=true; p<<s; return(ns)
+            else a<<c; ibase = @indent; state=':child'; next
+            end
+        when ':child'
+            case
+            when nl?; (s<<a if a.size>0); a=UString.new; state=':nl'; next
+            else a<<c; next
+            end
+        when ':1st'
+            case
+            when nl?; (s<<a if a.size>0); a=UString.new; state=':nl'; next
+            else a<<c; next
+            end
+        when '{eof}'
+            @fwd=true; (s<<a if a.size>0); a=UString.new; p<<s; return(ns)
+        when ':1st:ws'
+            case
+            when c==9,space?; next
+            when nl?; state=':nl'; next
+            else a<<c; state=':1st'; next
+            end
+        end
+      end
+    end
+
+  end
+end

data/lib/udon.rb

@@ -1,3 +1,19 @@
 module Udon
+  require 'udon/udon_parser'
+  VERSION = File.exist?(File.join(File.dirname(__FILE__),'VERSION')) ? File.read(File.join(File.dirname(__FILE__),'VERSION')) : ""
+  class << self
+    def version() VERSION end
 
+    def parse(source, opts={})
+      pp_ast = opts.delete(:pp_ast) || false
+      res = UdonParser::Parser.new(source, opts).parse
+      if pp_ast
+        require 'pp'
+        puts "\n----------------- AST ---------------------"
+        pp res
+        puts "\n-------------------------------------------"
+      end
+      return res
+    end
+  end
 end

data/machines/udon.machine

@@ -0,0 +1,42 @@
+ Main entrypoint - a complete udon document
+
++-----------------+----------+-----+----------------+-------------------------+
+| document()::[]  |          |     |                | :data_or_child          |
+| :data_or_child  | [\n]     |     | a<<<b          | :data                   |
+| :data_or_child  | [ ]      | b<< |                | :data_or_child          |
+| :data_or_child  | [#|]     |     | trash<<<b      | :child                  |
+| :data_or_child  | .        |     | a<<<b          | :data                   |
+
+| :data           | [\n]     | a<< | s<<<a          | :data_or_child          |
+| :data           | .        | a<< |                | :data                   |
+| :child          | [#]      |     | s<<<a          | comment(:data_or_child) |
+| {eof}           |          |     | a<<<b; s<<<a   | <done>                  |
+# TODO: lots more children
+
+ For error recovery (go to the next line)
+#+-----------------+----------+-----+----------------+--------------+
+#| to_nextline(ns) |          |     |                | :scan        |
+#| :scan           | {eof?}   |     |                | <ns>         |
+#| :scan           | [\n]     |     |                | <ns>         |
+#| :scan           | .        | <<  |                | :scan        |
+#+-----------------+----------+-----+----------------+--------------+
+
+ Block comments
++-----------------+-----------------+-----+-----------------+--------------+
+| comment(ns)::U  |                 |     | ipar=$indent    | :1st:ws      |
+:                 :                 :     : ibase=ipar+100  :              :
+| :1st:ws         | [ \t]           | <<  |                 | :1st:ws      |
+| :1st:ws         | [\n]            | <<  |                 | :nl          |
+| :1st:ws         | .               | a<< |                 | :1st         |
+| :1st            | [\n]            | <<  | s<<<a           | :nl          |
+| :1st            | .               | a<< |                 | :1st         |
+| :nl             | {$indent>ibase} |     |                 | :child       |
+| :nl             | [ \t\n]         | <<  |                 | :nl          |
+| :nl             | {$indent<=ipar} |     | p<<s            | <ns>         |
+| :nl             | .               | a<< | ibase = $indent | :child       |
+| :child          | [\n]            | <<  | s<<<a           | :nl          |
+| :child          | .               | a<< |                 | :child       |
+| {eof}           |                 |     | s<<<a; p<<s     | <ns>         |
++-----------------+-----------------+-----+-----------------+--------------+
+
+

data/test/helper.rb

@@ -1,3 +1,4 @@
+$KCODE = 'U'
 require 'rubygems'
 require 'bundler'
 begin
@@ -7,12 +8,83 @@ rescue Bundler::BundlerError => e
   $stderr.puts "Run `bundle install` to install missing gems"
   exit e.status_code
 end
-require 'test/unit'
-require 'shoulda'
+require 'iconv'
+require 'minitest/unit'
+require 'minitest/autorun'
+require 'minitest/pride'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
 require 'udon'
 
-class Test::Unit::TestCase
+class String
+  def udon; Udon.parse(self) end
+  def udon_pp; Udon.parse(self, :pp_ast=>true) end
 end
+
+class Array
+  def gcd; inject(){|n1,n2| n1.gcd(n2)} end
+end
+
+
+def gaussian(mean, stddev)
+  theta = 2 * Math::PI * rand
+  rho = Math.sqrt(-2 * Math.log(1 - rand))
+  scale = stddev * rho
+  if rand >= 0.5
+    return mean + scale * Math.cos(theta)
+  else
+    return mean + scale * Math.sin(theta)
+  end
+end
+
+def randstr(avg_length, char_dists = [[0.1,' '],[0.1,('A'..'Z')],[0.8,('a'..'z')]])
+  ret = ''
+  char_dists = char_dists.scan /./umn if char_dists.is_a?(String)
+  chrs = char_dists.sort_by{|n|n[0]}
+  chrs.map! do |k,ch|
+    if ch.nil?
+      ch = k
+      k = 1.0 / chrs.size.to_f
+    end
+    if ch.is_a?(Range)
+      if ch.max.is_a?(String)
+        [k, ((ch.min.unpack('U')[0])..(ch.max.unpack('U')[0]))]
+      else [k, ch] end
+    elsif ch.is_a?(Integer)
+      [k, (ch..ch)]
+    elsif ch.is_a?(String) && ch.scan(/./u).length == 1
+      [k, ((ch.unpack('U')[0])..(ch.unpack('U')[0]))]
+    elsif ch.is_a?(String)
+      [k, ch.unpack('U*')]
+    else
+      raise ArgumentError, 'Ranges and strings only'
+    end
+  end
+  len = [gaussian(avg_length,avg_length/3.0).round, 0].max
+  (0..len).each do
+    range_sel = rand
+    prob_sum = 0.0
+    chrs.each do |prob,cr|
+      prob_sum += prob
+      if prob_sum > range_sel
+        if cr.is_a?(Range)
+          ret << [(cr.min + rand(cr.max - cr.min + 1))].pack('U')
+        else
+          ret << cr[rand(cr.length)]
+        end
+        break
+      end
+    end
+  end
+  # (To fix iconv bug: cr http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/ )
+  ret = ret + ' '
+  Iconv.iconv('UTF-8//IGNORE', 'UTF-8', ret)
+  return ret[0..-2]
+end
+
+class MiniTest::Unit::TestCase
+end
+
+MiniTest::Unit.autorun

data/test/test_udon.rb

@@ -1,7 +1,35 @@
 require 'helper'
+$KCODE='U'
 
-class TestUdon < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+class TestUdon < MiniTest::Unit::TestCase
+  def test_blank_documents
+    assert_equal(''.udon,[])
+    (0..5).each do
+      s = randstr(200,"      \t\n\r")
+      assert_equal(s.udon.join(''),s)
+    end
+  end
+
+  def test_passthrough_documents
+    chars = [[0.10,  " \t"],
+             [0.05,  "\n\r"],
+             [0.40,  ('a'..'z')],
+             [0.15,  ('A'..'Z')],
+             [0.15,  ('0'..'9')],
+             [0.075, (32..126)],
+             [0.05,  (0..255)],
+             [0.025, (0..0xffff)]]
+    (0..5).each do
+      s = randstr(500,chars)
+      s.gsub! /^\s*(<\||#|\|)/u, ''
+      assert_equal(s.udon.join(''),s)
+    end
+  end
+
+  def test_block_comments
+    #leading = randstr(200,"      \t\n\r")
+    #comment = "# hello\na"
+    #following = randstr(200,"     \t\n\r")
+    #(leading + comment + following).udon_pp
   end
 end

data/udon.gemspec

@@ -0,0 +1,70 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{udon}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Joseph Wecker"]
+  s.date = %q{2011-08-18}
+  s.description = %q{Parse and generate udon, inspired by zml, haml, json, and more.}
+  s.email = %q{joseph.wecker@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/VERSION",
+    "lib/udon.rb",
+    "lib/udon/udon_parser.rb",
+    "machines/udon.machine",
+    "test/helper.rb",
+    "test/test_udon.rb",
+    "udon.gemspec"
+  ]
+  s.homepage = %q{http://udon.io}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Universal Document and Object Notation}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<minitest>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<reek>, ["~> 1.2.8"])
+      s.add_development_dependency(%q<roodi>, ["~> 2.1.0"])
+      s.add_development_dependency(%q<genmachine>, ["~> 0.1.6"])
+    else
+      s.add_dependency(%q<minitest>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<reek>, ["~> 1.2.8"])
+      s.add_dependency(%q<roodi>, ["~> 2.1.0"])
+      s.add_dependency(%q<genmachine>, ["~> 0.1.6"])
+    end
+  else
+    s.add_dependency(%q<minitest>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<reek>, ["~> 1.2.8"])
+    s.add_dependency(%q<roodi>, ["~> 2.1.0"])
+    s.add_dependency(%q<genmachine>, ["~> 0.1.6"])
+  end
+end
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: udon
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: 
   segments: 
   - 0
   - 0
-  - 0
-  version: 0.0.0
+  - 1
+  version: 0.0.1
 platform: ruby
 authors: 
 - Joseph Wecker
@@ -15,10 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-12 00:00:00 -07:00
+date: 2011-08-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -28,11 +29,11 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  name: shoulda
+  name: minitest
   version_requirements: *id001
   prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -47,8 +48,8 @@ dependencies:
   name: bundler
   version_requirements: *id002
   prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -63,8 +64,8 @@ dependencies:
   name: jeweler
   version_requirements: *id003
   prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -77,8 +78,8 @@ dependencies:
   name: rcov
   version_requirements: *id004
   prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -93,8 +94,8 @@ dependencies:
   name: reek
   version_requirements: *id005
   prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency 
+  type: :development
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -109,7 +110,22 @@ dependencies:
   name: roodi
   version_requirements: *id006
   prerelease: false
+- !ruby/object:Gem::Dependency 
   type: :development
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 0
+        - 1
+        - 6
+        version: 0.1.6
+  name: genmachine
+  version_requirements: *id007
+  prerelease: false
 description: Parse and generate udon, inspired by zml, haml, json, and more.
 email: joseph.wecker@gmail.com
 executables: []
@@ -126,9 +142,13 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- lib/VERSION
 - lib/udon.rb
+- lib/udon/udon_parser.rb
+- machines/udon.machine
 - test/helper.rb
 - test/test_udon.rb
+- udon.gemspec
 has_rdoc: true
 homepage: http://udon.io
 licenses: