liquidscript 0.3.1 → 0.4.0

data/lib/liquidscript/scanner/liquidscript.rb

@@ -0,0 +1,128 @@
+module Liquidscript
+  module Scanner
+    class Liquidscript < Base
+
+      include Base::DSL
+
+      attr_reader :tokens
+
+      define do
+        default_context :main
+
+        context :main do
+          set :number, %r{
+            -? [0-9][1-9]*     # the base of the number
+            (\.[0-9]+)?        # decmial portion, if needed
+            ([eE][+-]?[0-9]+)? # scientific notation
+          }x
+
+          set :string, %r{
+            '
+            [A-Za-z0-9_$\-]+
+            '?
+          }x
+
+          set :unops, %w(
+            !
+            ++
+            --
+            ~
+            new
+            return
+            typeof
+          )
+
+          set :binops, %w(
+            + - * / & | ^
+            << >> >>>
+            == ===
+            != !==
+            > >=
+            < <=
+            && ||
+            instanceof
+            or and
+          )
+
+          set :keywords, %w(
+            undefined
+            null
+            true
+            false
+          )
+
+          set :identifier, %r{[A-Za-z_$][A-Za-z0-9_$]*}
+
+          on("class")     {     emit :class         }
+          on("module")    {     emit :module        }
+          on("if")        {     emit :if            }
+          on("unless")    {     emit :unless        }
+          on("elsif")     {     emit :elsif         }
+          on("else")      {     emit :else          }
+          on(:number)     { |m| emit :number,  m    }
+          on(:string)     { |m| emit :sstring, m    }
+          on(:keywords)   { |m| emit :keyword, m    }
+          on(:unops)      { |m| emit :unop,    m    }
+          on("->")        {     emit :arrow         }
+          on("=")         {     emit :equal         }
+          on("{")         {     emit :lbrack        }
+          on("(")         {     emit :lparen        }
+          on("[")         {     emit :lbrace        }
+          on("}")         {     emit :rbrack        }
+          on(")")         {     emit :rparen        }
+          on("]")         {     emit :rbrace        }
+          on(":")         {     emit :colon         }
+          on(".")         {     emit :prop          }
+          on(",")         {     emit :comma         }
+          on("\n")        {     line!; emit :newline}
+          on(:binops)     { |m| emit :binop,   m    }
+          on(:identifier) { |m| emit :identifier, m }
+
+          on(/\"/ => :istring)
+          on(/#.*?\n/) { }
+          on(/[\s]/)   { }
+          on(:_) { error }
+        end
+
+        context :istring do
+          init do
+            @buffer = []
+          end
+
+          on(/\\"/) { |m| @buffer << m }
+          on(/"/) do
+            emit :istring, @buffer.join
+            exit
+          end
+
+          on(/\#\{(.*?)\}/) do |_, b|
+            emit :istring_begin, @buffer.join
+            lex :main => b
+            @buffer = []
+          end
+
+          on(:_) { |m| @buffer << m }
+        end
+      end
+
+      def initialize(*)
+        @line = 1
+        @cstart = 0
+        super
+      end
+
+      def line!
+        @line += 1
+        @cstart = @scanner.pos
+      end
+
+      def line
+        @line
+      end
+
+      def column
+        @scanner.pos - @cstart
+      end
+    end
+  end
+end

data/lib/liquidscript/scanner/token.rb

@@ -1,7 +1,7 @@
 require "liquidscript/icr/representable"
 
 module Liquidscript
-  class Scanner
+  module Scanner
     class Token
 
       attr_accessor :type
@@ -16,11 +16,7 @@ module Liquidscript
         @type   = type
         @line   = line
         @column = column
-        @value  = begin
-          value.pack("c*")
-        rescue NoMethodError, TypeError
-          value
-        end
+        @value  = value
       end
 
       def to_a

data/lib/liquidscript/template.rb

@@ -9,7 +9,6 @@ module Liquidscript
 
     def render
       @_render ||= begin
-        p Scanner.new(@data).each
         compiler = Compiler::ICR.new(Scanner.new(@data))
         compiler.compile
         puts ICR::Sexp.new(compiler.top).output

data/lib/liquidscript/version.rb

@@ -1,5 +1,5 @@
 module Liquidscript
 
   # The current version of liquidscript.
-  VERSION = "0.3.1".freeze
+  VERSION = "0.4.0".freeze
 end

data/spec/fixtures/complex.generate.yml

@@ -1,4 +1,5 @@
 data: |
+  # comment
   thing = {
     test: -> {
       if(undefined) {

data/spec/fixtures/main.compile.yml

@@ -30,7 +30,7 @@ compiled:
               - console
             - - :identifier
               - log
-          - - :dstring
+          - - :istring
             - hello world
         - - :newline
   - - :newline

data/spec/fixtures/string.compile.yml

@@ -0,0 +1,15 @@
+data: |
+  "hello #{console}"
+compiled:
+  - :exec
+  - - :_context
+    - []
+  - - :interop
+    - - :istring_begin
+      - "hello "
+    - - :get
+      - - :_variable
+        - :console
+    - - :istring
+      - ""
+  - - :newline

data/spec/fixtures/string.generate.yml

@@ -3,9 +3,11 @@ data: |
   foo  = 'world
   thing = "hello \" test
   world"
+  interop = "hello #{foo}"
 
 compiled: |
-  var test, foo, thing;
+  var test, foo, thing, interop;
   test = "hello";
   foo = 'world';
   thing = "hello \" test \n world";
+  interop = ["hello ", foo, ""].join('');

data/spec/{lib/liquidscript → liquidscript}/generator/javascript_spec.rb

@@ -1,14 +1,19 @@
 require "spec_helper"
 require "yaml"
 
-describe Liquidscript::Generator::Javascript do
+describe Generator::Javascript do
   describe "with fixtures" do
     Dir.glob("spec/fixtures/*.generate.yml") do |file|
       content = YAML.load_file file
       file =~ /spec\/fixtures\/(.*)\.generate\.yml/
 
       it "generates #{$1}" do
-        expect(content["data"]).to generate(content["compiled"])
+        begin
+          expect(content["data"]).to generate(content["compiled"])
+        rescue Error
+          p Scanner::Liquidscript.new(content["data"]).each.to_a
+          raise
+        end
       end
     end
   end

data/spec/{lib/liquidscript → liquidscript}/scanner/lexer_spec.rb

@@ -1,14 +1,6 @@
 require "spec_helper"
 
-describe Liquidscript::Scanner::Lexer, :lexer_helper do
-  subject { described_class.new }
-  describe "#emit" do
-    it "pushes a token" do
-      subject.instance_exec { @data = "hello"; @ts = 0; @te = 6 }
-      subject.emit(:test)
-      expect(subject.tokens.first).to be_token(:test, "hello")
-    end
-  end
+describe Liquidscript::Scanner::Liquidscript, :lexer_helper do
 
   describe "#perform" do
     it "scans a number" do
@@ -19,29 +11,35 @@ describe Liquidscript::Scanner::Lexer, :lexer_helper do
 
     it "scans a string" do
       scan('"hello world" ').should eq [
-        [:dstring, '"hello world"']
+        [:istring, 'hello world']
       ]
 
       scan(" 'foobar").should eq [
         [:sstring, "'foobar"]
       ]
+
+      scan('"hello #{world}"').should eq [
+        [:istring_begin, "hello "],
+        [:identifier, "world"],
+        [:istring, ""]
+      ]
     end
 
     it "scans an identifier" do
       scan('test = 4').should eq [
         [:identifier, "test"],
-        [:equal, "="],
+        [:equal, nil],
         [:number, "4"]
       ]
     end
 
     it "scans brackets" do
       scan("{ test = 3 }").should eq [
-        [:lbrack, "{"],
+        [:lbrack, nil],
         [:identifier, "test"],
-        [:equal, "="],
+        [:equal, nil],
         [:number, "3"],
-        [:rbrack, "}"]
+        [:rbrack, nil]
       ]
     end
 
@@ -49,7 +47,7 @@ describe Liquidscript::Scanner::Lexer, :lexer_helper do
       scan("return test = new foo").should eq [
         [:unop, "return"],
         [:identifier, "test"],
-        [:equal, "="],
+        [:equal, nil],
         [:unop, "new"],
         [:identifier, "foo"]
       ]

data/spec/{lib/liquidscript → liquidscript}/scanner_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 describe Liquidscript::Scanner do
-  subject { described_class.new("42") }
+  subject { described_class::Liquidscript.new("42") }
   it { should be_a Enumerable }
 
   describe "#each" do
@@ -10,7 +10,7 @@ describe Liquidscript::Scanner do
   end
 
   context "with invalid input" do
-    subject { described_class.new("'test'") }
+    subject { described_class::Liquidscript.new("';") }
 
     it "raises an error" do
       expect {

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+require "rubygems"
 require "bundler/setup"
 Bundler.setup
 

data/spec/support/helpers/lexer_helper.rb

@@ -1,5 +1,5 @@
 module LexerHelper
   def scan(source)
-    subject.perform(source)
+    described_class.new(source).scan.tokens
   end
 end

data/spec/support/matchers/compile.rb

@@ -36,6 +36,6 @@ RSpec::Matchers.define :compile do
   end
 
   def compiler(data)
-    Compiler::ICR.new(Scanner.new(data))
+    Compiler::ICR.new(Scanner::Liquidscript.new(data))
   end
 end

data/spec/support/matchers/generate.rb

@@ -29,13 +29,13 @@ RSpec::Matchers.define :generate do |v|
   diffable
 
   def generator(data)
-    compiler = Compiler::ICR.new(s = Scanner.new(data))
+    compiler = Compiler::ICR.new(s = Scanner::Liquidscript.new(data))
     compiler.compile
     Generator::Javascript.new(compiler.top)
   end
 
   def tree(data)
-    compiler = Compiler::ICR.new(Scanner.new(data))
+    compiler = Compiler::ICR.new(Scanner::Liquidscript.new(data))
     compiler.compile
     compiler.top.to_sexp
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: liquidscript
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Jeremy Rodi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-09 00:00:00.000000000 Z
+date: 2014-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,8 +130,12 @@ files:
 - lib/liquidscript/icr/sexp.rb
 - lib/liquidscript/icr/variable.rb
 - lib/liquidscript/scanner.rb
-- lib/liquidscript/scanner/lexer.rb
-- lib/liquidscript/scanner/lexer.rl
+- lib/liquidscript/scanner/base.rb
+- lib/liquidscript/scanner/base/builder.rb
+- lib/liquidscript/scanner/base/context.rb
+- lib/liquidscript/scanner/base/dsl.rb
+- lib/liquidscript/scanner/base/lexer.rb
+- lib/liquidscript/scanner/liquidscript.rb
 - lib/liquidscript/scanner/token.rb
 - lib/liquidscript/template.rb
 - lib/liquidscript/version.rb
@@ -148,17 +152,18 @@ files:
 - spec/fixtures/main.compile.yml
 - spec/fixtures/operator.generate.yml
 - spec/fixtures/set.generate.yml
+- spec/fixtures/string.compile.yml
 - spec/fixtures/string.generate.yml
 - spec/fixtures/underscore.ls
-- spec/lib/liquidscript/buffer_spec.rb
-- spec/lib/liquidscript/compiler/icr_spec.rb
-- spec/lib/liquidscript/generator/javascript_spec.rb
-- spec/lib/liquidscript/icr/code_spec.rb
-- spec/lib/liquidscript/icr/context_spec.rb
-- spec/lib/liquidscript/icr/set_spec.rb
-- spec/lib/liquidscript/scanner/lexer_spec.rb
-- spec/lib/liquidscript/scanner/token_spec.rb
-- spec/lib/liquidscript/scanner_spec.rb
+- spec/liquidscript/buffer_spec.rb
+- spec/liquidscript/compiler/icr_spec.rb
+- spec/liquidscript/generator/javascript_spec.rb
+- spec/liquidscript/icr/code_spec.rb
+- spec/liquidscript/icr/context_spec.rb
+- spec/liquidscript/icr/set_spec.rb
+- spec/liquidscript/scanner/lexer_spec.rb
+- spec/liquidscript/scanner/token_spec.rb
+- spec/liquidscript/scanner_spec.rb
 - spec/spec_helper.rb
 - spec/support/helpers/lexer_helper.rb
 - spec/support/matchers/be_token.rb
@@ -200,17 +205,18 @@ test_files:
 - spec/fixtures/main.compile.yml
 - spec/fixtures/operator.generate.yml
 - spec/fixtures/set.generate.yml
+- spec/fixtures/string.compile.yml
 - spec/fixtures/string.generate.yml
 - spec/fixtures/underscore.ls
-- spec/lib/liquidscript/buffer_spec.rb
-- spec/lib/liquidscript/compiler/icr_spec.rb
-- spec/lib/liquidscript/generator/javascript_spec.rb
-- spec/lib/liquidscript/icr/code_spec.rb
-- spec/lib/liquidscript/icr/context_spec.rb
-- spec/lib/liquidscript/icr/set_spec.rb
-- spec/lib/liquidscript/scanner/lexer_spec.rb
-- spec/lib/liquidscript/scanner/token_spec.rb
-- spec/lib/liquidscript/scanner_spec.rb
+- spec/liquidscript/buffer_spec.rb
+- spec/liquidscript/compiler/icr_spec.rb
+- spec/liquidscript/generator/javascript_spec.rb
+- spec/liquidscript/icr/code_spec.rb
+- spec/liquidscript/icr/context_spec.rb
+- spec/liquidscript/icr/set_spec.rb
+- spec/liquidscript/scanner/lexer_spec.rb
+- spec/liquidscript/scanner/token_spec.rb
+- spec/liquidscript/scanner_spec.rb
 - spec/spec_helper.rb
 - spec/support/helpers/lexer_helper.rb
 - spec/support/matchers/be_token.rb