locomotivecms-liquid 2.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/History.md +108 -0
  3. data/MIT-LICENSE +20 -0
  4. data/README.md +75 -0
  5. data/lib/extras/liquid_view.rb +51 -0
  6. data/lib/liquid/block.rb +160 -0
  7. data/lib/liquid/condition.rb +120 -0
  8. data/lib/liquid/context.rb +268 -0
  9. data/lib/liquid/document.rb +18 -0
  10. data/lib/liquid/drop.rb +74 -0
  11. data/lib/liquid/errors.rb +21 -0
  12. data/lib/liquid/extensions.rb +62 -0
  13. data/lib/liquid/file_system.rb +62 -0
  14. data/lib/liquid/htmltags.rb +74 -0
  15. data/lib/liquid/i18n.rb +39 -0
  16. data/lib/liquid/interrupts.rb +17 -0
  17. data/lib/liquid/lexer.rb +51 -0
  18. data/lib/liquid/locales/en.yml +25 -0
  19. data/lib/liquid/module_ex.rb +62 -0
  20. data/lib/liquid/parser.rb +89 -0
  21. data/lib/liquid/standardfilters.rb +285 -0
  22. data/lib/liquid/strainer.rb +53 -0
  23. data/lib/liquid/tag.rb +61 -0
  24. data/lib/liquid/tags/assign.rb +36 -0
  25. data/lib/liquid/tags/break.rb +21 -0
  26. data/lib/liquid/tags/capture.rb +40 -0
  27. data/lib/liquid/tags/case.rb +77 -0
  28. data/lib/liquid/tags/comment.rb +16 -0
  29. data/lib/liquid/tags/continue.rb +21 -0
  30. data/lib/liquid/tags/cycle.rb +61 -0
  31. data/lib/liquid/tags/decrement.rb +39 -0
  32. data/lib/liquid/tags/default_content.rb +21 -0
  33. data/lib/liquid/tags/extends.rb +79 -0
  34. data/lib/liquid/tags/for.rb +167 -0
  35. data/lib/liquid/tags/if.rb +100 -0
  36. data/lib/liquid/tags/ifchanged.rb +20 -0
  37. data/lib/liquid/tags/include.rb +97 -0
  38. data/lib/liquid/tags/increment.rb +36 -0
  39. data/lib/liquid/tags/inherited_block.rb +101 -0
  40. data/lib/liquid/tags/raw.rb +22 -0
  41. data/lib/liquid/tags/unless.rb +33 -0
  42. data/lib/liquid/template.rb +213 -0
  43. data/lib/liquid/utils.rb +30 -0
  44. data/lib/liquid/variable.rb +109 -0
  45. data/lib/liquid/version.rb +4 -0
  46. data/lib/liquid.rb +72 -0
  47. data/lib/locomotivecms-liquid.rb +1 -0
  48. metadata +94 -0
@@ -0,0 +1,268 @@
1
+ module Liquid
2
+
3
+ # Context keeps the variable stack and resolves variables, as well as keywords
4
+ #
5
+ # context['variable'] = 'testing'
6
+ # context['variable'] #=> 'testing'
7
+ # context['true'] #=> true
8
+ # context['10.2232'] #=> 10.2232
9
+ #
10
+ # context.stack do
11
+ # context['bob'] = 'bobsen'
12
+ # end
13
+ #
14
+ # context['bob'] #=> nil class Context
15
+ class Context
16
+ attr_reader :scopes, :errors, :registers, :environments, :resource_limits
17
+
18
+ def initialize(environments = {}, outer_scope = {}, registers = {}, rethrow_errors = false, resource_limits = {})
19
+ @environments = [environments].flatten
20
+ @scopes = [(outer_scope || {})]
21
+ @registers = registers
22
+ @errors = []
23
+ @rethrow_errors = rethrow_errors
24
+ @resource_limits = (resource_limits || {}).merge!({ :render_score_current => 0, :assign_score_current => 0 })
25
+ squash_instance_assigns_with_environments
26
+
27
+ @interrupts = []
28
+ end
29
+
30
+ def resource_limits_reached?
31
+ (@resource_limits[:render_length_limit] && @resource_limits[:render_length_current] > @resource_limits[:render_length_limit]) ||
32
+ (@resource_limits[:render_score_limit] && @resource_limits[:render_score_current] > @resource_limits[:render_score_limit] ) ||
33
+ (@resource_limits[:assign_score_limit] && @resource_limits[:assign_score_current] > @resource_limits[:assign_score_limit] )
34
+ end
35
+
36
+ def strainer
37
+ @strainer ||= Strainer.create(self)
38
+ end
39
+
40
+ # Adds filters to this context.
41
+ #
42
+ # Note that this does not register the filters with the main Template object. see <tt>Template.register_filter</tt>
43
+ # for that
44
+ def add_filters(filters)
45
+ filters = [filters].flatten.compact
46
+
47
+ filters.each do |f|
48
+ raise ArgumentError, "Expected module but got: #{f.class}" unless f.is_a?(Module)
49
+ Strainer.add_known_filter(f)
50
+ strainer.extend(f)
51
+ end
52
+ end
53
+
54
+ # are there any not handled interrupts?
55
+ def has_interrupt?
56
+ @interrupts.any?
57
+ end
58
+
59
+ # push an interrupt to the stack. this interrupt is considered not handled.
60
+ def push_interrupt(e)
61
+ @interrupts.push(e)
62
+ end
63
+
64
+ # pop an interrupt from the stack
65
+ def pop_interrupt
66
+ @interrupts.pop
67
+ end
68
+
69
+ def handle_error(e)
70
+ errors.push(e)
71
+ raise if @rethrow_errors
72
+
73
+ case e
74
+ when SyntaxError
75
+ "Liquid syntax error: #{e.message}"
76
+ else
77
+ "Liquid error: #{e.message}"
78
+ end
79
+ end
80
+
81
+ def invoke(method, *args)
82
+ strainer.invoke(method, *args)
83
+ end
84
+
85
+ # Push new local scope on the stack. use <tt>Context#stack</tt> instead
86
+ def push(new_scope={})
87
+ @scopes.unshift(new_scope)
88
+ raise StackLevelError, "Nesting too deep" if @scopes.length > 100
89
+ end
90
+
91
+ # Merge a hash of variables in the current local scope
92
+ def merge(new_scopes)
93
+ @scopes[0].merge!(new_scopes)
94
+ end
95
+
96
+ # Pop from the stack. use <tt>Context#stack</tt> instead
97
+ def pop
98
+ raise ContextError if @scopes.size == 1
99
+ @scopes.shift
100
+ end
101
+
102
+ # Pushes a new local scope on the stack, pops it at the end of the block
103
+ #
104
+ # Example:
105
+ # context.stack do
106
+ # context['var'] = 'hi'
107
+ # end
108
+ #
109
+ # context['var] #=> nil
110
+ def stack(new_scope={})
111
+ push(new_scope)
112
+ yield
113
+ ensure
114
+ pop
115
+ end
116
+
117
+ def clear_instance_assigns
118
+ @scopes[0] = {}
119
+ end
120
+
121
+ # Only allow String, Numeric, Hash, Array, Proc, Boolean or <tt>Liquid::Drop</tt>
122
+ def []=(key, value)
123
+ @scopes[0][key] = value
124
+ end
125
+
126
+ def [](key)
127
+ resolve(key)
128
+ end
129
+
130
+ def has_key?(key)
131
+ resolve(key) != nil
132
+ end
133
+
134
+ private
135
+
136
+ LITERALS = {
137
+ nil => nil, 'nil' => nil, 'null' => nil, '' => nil,
138
+ 'true' => true,
139
+ 'false' => false,
140
+ 'blank' => :blank?,
141
+ 'empty' => :empty?
142
+ }
143
+
144
+ # Look up variable, either resolve directly after considering the name. We can directly handle
145
+ # Strings, digits, floats and booleans (true,false).
146
+ # If no match is made we lookup the variable in the current scope and
147
+ # later move up to the parent blocks to see if we can resolve the variable somewhere up the tree.
148
+ # Some special keywords return symbols. Those symbols are to be called on the rhs object in expressions
149
+ #
150
+ # Example:
151
+ # products == empty #=> products.empty?
152
+ def resolve(key)
153
+ if LITERALS.key?(key)
154
+ LITERALS[key]
155
+ else
156
+ case key
157
+ when /^'(.*)'$/ # Single quoted strings
158
+ $1
159
+ when /^"(.*)"$/ # Double quoted strings
160
+ $1
161
+ when /^(-?\d+)$/ # Integer and floats
162
+ $1.to_i
163
+ when /^\((\S+)\.\.(\S+)\)$/ # Ranges
164
+ (resolve($1).to_i..resolve($2).to_i)
165
+ when /^(-?\d[\d\.]+)$/ # Floats
166
+ $1.to_f
167
+ else
168
+ variable(key)
169
+ end
170
+ end
171
+ end
172
+
173
+ # Fetches an object starting at the local scope and then moving up the hierachy
174
+ def find_variable(key)
175
+ scope = @scopes.find { |s| s.has_key?(key) }
176
+ variable = nil
177
+
178
+ if scope.nil?
179
+ @environments.each do |e|
180
+ if variable = lookup_and_evaluate(e, key)
181
+ scope = e
182
+ break
183
+ end
184
+ end
185
+ end
186
+
187
+ scope ||= @environments.last || @scopes.last
188
+ variable ||= lookup_and_evaluate(scope, key)
189
+
190
+ variable = variable.to_liquid
191
+ variable.context = self if variable.respond_to?(:context=)
192
+
193
+ return variable
194
+ end
195
+
196
+ # Resolves namespaced queries gracefully.
197
+ #
198
+ # Example
199
+ # @context['hash'] = {"name" => 'tobi'}
200
+ # assert_equal 'tobi', @context['hash.name']
201
+ # assert_equal 'tobi', @context['hash["name"]']
202
+ def variable(markup)
203
+ parts = markup.scan(VariableParser)
204
+ square_bracketed = /^\[(.*)\]$/
205
+
206
+ first_part = parts.shift
207
+
208
+ if first_part =~ square_bracketed
209
+ first_part = resolve($1)
210
+ end
211
+
212
+ if object = find_variable(first_part)
213
+
214
+ parts.each do |part|
215
+ part = resolve($1) if part_resolved = (part =~ square_bracketed)
216
+
217
+ # If object is a hash- or array-like object we look for the
218
+ # presence of the key and if its available we return it
219
+ if object.respond_to?(:[]) and
220
+ ((object.respond_to?(:has_key?) and object.has_key?(part)) or
221
+ (object.respond_to?(:fetch) and part.is_a?(Integer)))
222
+
223
+ # if its a proc we will replace the entry with the proc
224
+ res = lookup_and_evaluate(object, part)
225
+ object = res.to_liquid
226
+
227
+ # Some special cases. If the part wasn't in square brackets and
228
+ # no key with the same name was found we interpret following calls
229
+ # as commands and call them on the current object
230
+ elsif !part_resolved and object.respond_to?(part) and ['size', 'first', 'last'].include?(part)
231
+
232
+ object = object.send(part.intern).to_liquid
233
+
234
+ # No key was present with the desired value and it wasn't one of the directly supported
235
+ # keywords either. The only thing we got left is to return nil
236
+ else
237
+ return nil
238
+ end
239
+
240
+ # If we are dealing with a drop here we have to
241
+ object.context = self if object.respond_to?(:context=)
242
+ end
243
+ end
244
+
245
+ object
246
+ end # variable
247
+
248
+ def lookup_and_evaluate(obj, key)
249
+ if (value = obj[key]).is_a?(Proc) && obj.respond_to?(:[]=)
250
+ obj[key] = (value.arity == 0) ? value.call : value.call(self)
251
+ else
252
+ value
253
+ end
254
+ end # lookup_and_evaluate
255
+
256
+ def squash_instance_assigns_with_environments
257
+ @scopes.last.each_key do |k|
258
+ @environments.each do |env|
259
+ if env.has_key?(k)
260
+ scopes.last[k] = lookup_and_evaluate(env, k)
261
+ break
262
+ end
263
+ end
264
+ end
265
+ end # squash_instance_assigns_with_environments
266
+ end # Context
267
+
268
+ end # Liquid
@@ -0,0 +1,18 @@
1
+ module Liquid
2
+ class Document < Block
3
+ # we don't need markup to open this block
4
+ def initialize(tokens, options = {})
5
+ @options = options
6
+ parse(tokens)
7
+ end
8
+
9
+ # There isn't a real delimiter
10
+ def block_delimiter
11
+ []
12
+ end
13
+
14
+ # Document blocks don't need to be terminated since they are not actually opened
15
+ def assert_missing_delimitation!
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,74 @@
1
+ require 'set'
2
+
3
+ module Liquid
4
+
5
+ # A drop in liquid is a class which allows you to export DOM like things to liquid.
6
+ # Methods of drops are callable.
7
+ # The main use for liquid drops is to implement lazy loaded objects.
8
+ # If you would like to make data available to the web designers which you don't want loaded unless needed then
9
+ # a drop is a great way to do that.
10
+ #
11
+ # Example:
12
+ #
13
+ # class ProductDrop < Liquid::Drop
14
+ # def top_sales
15
+ # Shop.current.products.find(:all, :order => 'sales', :limit => 10 )
16
+ # end
17
+ # end
18
+ #
19
+ # tmpl = Liquid::Template.parse( ' {% for product in product.top_sales %} {{ product.name }} {%endfor%} ' )
20
+ # tmpl.render('product' => ProductDrop.new ) # will invoke top_sales query.
21
+ #
22
+ # Your drop can either implement the methods sans any parameters or implement the before_method(name) method which is a
23
+ # catch all.
24
+ class Drop
25
+ attr_writer :context
26
+
27
+ EMPTY_STRING = ''.freeze
28
+
29
+ # Catch all for the method
30
+ def before_method(method)
31
+ nil
32
+ end
33
+
34
+ # called by liquid to invoke a drop
35
+ def invoke_drop(method_or_key)
36
+ if method_or_key && method_or_key != EMPTY_STRING && self.class.invokable?(method_or_key)
37
+ send(method_or_key)
38
+ else
39
+ before_method(method_or_key)
40
+ end
41
+ end
42
+
43
+ def has_key?(name)
44
+ true
45
+ end
46
+
47
+ def inspect
48
+ self.class.to_s
49
+ end
50
+
51
+ def to_liquid
52
+ self
53
+ end
54
+
55
+ alias :[] :invoke_drop
56
+
57
+ private
58
+
59
+ # Check for method existence without invoking respond_to?, which creates symbols
60
+ def self.invokable?(method_name)
61
+ unless @invokable_methods
62
+ # Ruby 1.8 compatibility: call to_s on method names (which are strings in 1.8, but already symbols in 1.9)
63
+ blacklist = (Liquid::Drop.public_instance_methods + [:each]).map(&:to_s)
64
+ if include?(Enumerable)
65
+ blacklist += Enumerable.public_instance_methods.map(&:to_s)
66
+ blacklist -= [:sort, :count, :first, :min, :max, :include?].map(&:to_s)
67
+ end
68
+ whitelist = [:to_liquid] + (public_instance_methods.map(&:to_s) - blacklist.map(&:to_s))
69
+ @invokable_methods = Set.new(whitelist.map(&:to_s))
70
+ end
71
+ @invokable_methods.include?(method_name.to_s)
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,21 @@
1
+ module Liquid
2
+ class Error < ::StandardError
3
+
4
+ attr_accessor :line
5
+
6
+ def initialize(message = nil, line = nil)
7
+ @line = line
8
+ super(message)
9
+ end
10
+
11
+ end
12
+
13
+ class ArgumentError < Error; end
14
+ class ContextError < Error; end
15
+ class FilterNotFound < Error; end
16
+ class FileSystemError < Error; end
17
+ class StandardError < Error; end
18
+ class SyntaxError < Error; end
19
+ class StackLevelError < Error; end
20
+ class MemoryError < Error; end
21
+ end
@@ -0,0 +1,62 @@
1
+ require 'time'
2
+ require 'date'
3
+
4
+ class String # :nodoc:
5
+ def to_liquid
6
+ self
7
+ end
8
+ end
9
+
10
+ class Array # :nodoc:
11
+ def to_liquid
12
+ self
13
+ end
14
+ end
15
+
16
+ class Hash # :nodoc:
17
+ def to_liquid
18
+ self
19
+ end
20
+ end
21
+
22
+ class Numeric # :nodoc:
23
+ def to_liquid
24
+ self
25
+ end
26
+ end
27
+
28
+ class Time # :nodoc:
29
+ def to_liquid
30
+ self
31
+ end
32
+ end
33
+
34
+ class DateTime < Date # :nodoc:
35
+ def to_liquid
36
+ self
37
+ end
38
+ end
39
+
40
+ class Date # :nodoc:
41
+ def to_liquid
42
+ self
43
+ end
44
+ end
45
+
46
+ class TrueClass
47
+ def to_liquid # :nodoc:
48
+ self
49
+ end
50
+ end
51
+
52
+ class FalseClass
53
+ def to_liquid # :nodoc:
54
+ self
55
+ end
56
+ end
57
+
58
+ class NilClass
59
+ def to_liquid # :nodoc:
60
+ self
61
+ end
62
+ end
@@ -0,0 +1,62 @@
1
+ module Liquid
2
+ # A Liquid file system is a way to let your templates retrieve other templates for use with the include tag.
3
+ #
4
+ # You can implement subclasses that retrieve templates from the database, from the file system using a different
5
+ # path structure, you can provide them as hard-coded inline strings, or any manner that you see fit.
6
+ #
7
+ # You can add additional instance variables, arguments, or methods as needed.
8
+ #
9
+ # Example:
10
+ #
11
+ # Liquid::Template.file_system = Liquid::LocalFileSystem.new(template_path)
12
+ # liquid = Liquid::Template.parse(template)
13
+ #
14
+ # This will parse the template with a LocalFileSystem implementation rooted at 'template_path'.
15
+ class BlankFileSystem
16
+ # Called by Liquid to retrieve a template file
17
+ def read_template_file(template_path, context)
18
+ raise FileSystemError, "This liquid context does not allow includes."
19
+ end
20
+ end
21
+
22
+ # This implements an abstract file system which retrieves template files named in a manner similar to Rails partials,
23
+ # ie. with the template name prefixed with an underscore. The extension ".liquid" is also added.
24
+ #
25
+ # For security reasons, template paths are only allowed to contain letters, numbers, and underscore.
26
+ #
27
+ # Example:
28
+ #
29
+ # file_system = Liquid::LocalFileSystem.new("/some/path")
30
+ #
31
+ # file_system.full_path("mypartial") # => "/some/path/_mypartial.liquid"
32
+ # file_system.full_path("dir/mypartial") # => "/some/path/dir/_mypartial.liquid"
33
+ #
34
+ class LocalFileSystem
35
+ attr_accessor :root
36
+
37
+ def initialize(root)
38
+ @root = root
39
+ end
40
+
41
+ def read_template_file(template_path, context)
42
+ full_path = full_path(template_path)
43
+ raise FileSystemError, "No such template '#{template_path}'" unless File.exists?(full_path)
44
+
45
+ File.read(full_path)
46
+ end
47
+
48
+ def full_path(template_path)
49
+ raise FileSystemError, "Illegal template name '#{template_path}'" unless template_path =~ /^[^.\/][a-zA-Z0-9_\/]+$/
50
+
51
+ full_path = if template_path.include?('/')
52
+ File.join(root, File.dirname(template_path), "_#{File.basename(template_path)}.liquid")
53
+ else
54
+ File.join(root, "_#{template_path}.liquid")
55
+ end
56
+
57
+ raise FileSystemError, "Illegal template path '#{File.expand_path(full_path)}'" unless File.expand_path(full_path) =~ /^#{File.expand_path(root)}/
58
+
59
+ full_path
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,74 @@
1
+ module Liquid
2
+ class TableRow < Block
3
+ Syntax = /(\w+)\s+in\s+(#{QuotedFragment}+)/o
4
+
5
+ def initialize(tag_name, markup, tokens, options)
6
+ if markup =~ Syntax
7
+ @variable_name = $1
8
+ @collection_name = $2
9
+ @attributes = {}
10
+ markup.scan(TagAttributes) do |key, value|
11
+ @attributes[key] = value
12
+ end
13
+ else
14
+ raise SyntaxError.new(options[:locale].t("errors.syntax.table_row"), options[:line])
15
+ end
16
+
17
+ super
18
+ end
19
+
20
+ def render(context)
21
+ collection = context[@collection_name] or return ''
22
+
23
+ from = @attributes['offset'] ? context[@attributes['offset']].to_i : 0
24
+ to = @attributes['limit'] ? from + context[@attributes['limit']].to_i : nil
25
+
26
+ collection = Utils.slice_collection_using_each(collection, from, to)
27
+
28
+ length = collection.length
29
+
30
+ cols = context[@attributes['cols']].to_i
31
+
32
+ row = 1
33
+ col = 0
34
+
35
+ result = "<tr class=\"row1\">\n"
36
+ context.stack do
37
+
38
+ collection.each_with_index do |item, index|
39
+ context[@variable_name] = item
40
+ context['tablerowloop'] = {
41
+ 'length' => length,
42
+ 'index' => index + 1,
43
+ 'index0' => index,
44
+ 'col' => col + 1,
45
+ 'col0' => col,
46
+ 'index0' => index,
47
+ 'rindex' => length - index,
48
+ 'rindex0' => length - index - 1,
49
+ 'first' => (index == 0),
50
+ 'last' => (index == length - 1),
51
+ 'col_first' => (col == 0),
52
+ 'col_last' => (col == cols - 1)
53
+ }
54
+
55
+
56
+ col += 1
57
+
58
+ result << "<td class=\"col#{col}\">" << render_all(@nodelist, context) << '</td>'
59
+
60
+ if col == cols and (index != length - 1)
61
+ col = 0
62
+ row += 1
63
+ result << "</tr>\n<tr class=\"row#{row}\">"
64
+ end
65
+
66
+ end
67
+ end
68
+ result << "</tr>\n"
69
+ result
70
+ end
71
+ end
72
+
73
+ Template.register_tag('tablerow', TableRow)
74
+ end
@@ -0,0 +1,39 @@
1
+ require 'yaml'
2
+
3
+ module Liquid
4
+ class I18n
5
+ DEFAULT_LOCALE = File.join(File.expand_path(File.dirname(__FILE__)), "locales", "en.yml")
6
+
7
+ class TranslationError < StandardError
8
+ end
9
+
10
+ attr_reader :path
11
+
12
+ def initialize(path = DEFAULT_LOCALE)
13
+ @path = path
14
+ end
15
+
16
+ def translate(name, vars = {})
17
+ interpolate(deep_fetch_translation(name), vars)
18
+ end
19
+ alias_method :t, :translate
20
+
21
+ def locale
22
+ @locale ||= YAML.load_file(@path)
23
+ end
24
+
25
+ private
26
+ def interpolate(name, vars)
27
+ name.gsub(/%\{(\w+)\}/) {
28
+ # raise TranslationError, "Undefined key #{$1} for interpolation in translation #{name}" unless vars[$1.to_sym]
29
+ "#{vars[$1.to_sym]}"
30
+ }
31
+ end
32
+
33
+ def deep_fetch_translation(name)
34
+ name.split('.').reduce(locale) do |level, cur|
35
+ level[cur] or raise TranslationError, "Translation for #{name} does not exist in locale #{path}"
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,17 @@
1
+ module Liquid
2
+
3
+ # An interrupt is any command that breaks processing of a block (ex: a for loop).
4
+ class Interrupt
5
+ attr_reader :message
6
+
7
+ def initialize(message=nil)
8
+ @message = message || "interrupt"
9
+ end
10
+ end
11
+
12
+ # Interrupt that is thrown whenever a {% break %} is called.
13
+ class BreakInterrupt < Interrupt; end
14
+
15
+ # Interrupt that is thrown whenever a {% continue %} is called.
16
+ class ContinueInterrupt < Interrupt; end
17
+ end
@@ -0,0 +1,51 @@
1
+ require "strscan"
2
+ module Liquid
3
+ class Lexer
4
+ SPECIALS = {
5
+ '|' => :pipe,
6
+ '.' => :dot,
7
+ ':' => :colon,
8
+ ',' => :comma,
9
+ '[' => :open_square,
10
+ ']' => :close_square,
11
+ '(' => :open_round,
12
+ ')' => :close_round
13
+ }
14
+ IDENTIFIER = /[\w\-?!]+/
15
+ SINGLE_STRING_LITERAL = /'[^\']*'/
16
+ DOUBLE_STRING_LITERAL = /"[^\"]*"/
17
+ NUMBER_LITERAL = /-?\d+(\.\d+)?/
18
+ DOTDOT = /\.\./
19
+ COMPARISON_OPERATOR = /==|!=|<>|<=?|>=?|contains/
20
+
21
+ def initialize(input)
22
+ @ss = StringScanner.new(input.rstrip)
23
+ end
24
+
25
+ def tokenize
26
+ @output = []
27
+
28
+ while !@ss.eos?
29
+ @ss.skip(/\s*/)
30
+ tok = case
31
+ when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
32
+ when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
33
+ when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
34
+ when t = @ss.scan(NUMBER_LITERAL) then [:number, t]
35
+ when t = @ss.scan(IDENTIFIER) then [:id, t]
36
+ when t = @ss.scan(DOTDOT) then [:dotdot, t]
37
+ else
38
+ c = @ss.getch
39
+ if s = SPECIALS[c]
40
+ [s,c]
41
+ else
42
+ raise SyntaxError, "Unexpected character #{c}"
43
+ end
44
+ end
45
+ @output << tok
46
+ end
47
+
48
+ @output << [:end_of_string]
49
+ end
50
+ end
51
+ end