predicator 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +42 -0
- data/{LICENSE.txt → LICENSE} +1 -1
- data/README.md +322 -46
- data/lib/predicator/ast.rb +74 -187
- data/lib/predicator/boundary.rb +237 -0
- data/lib/predicator/civil.rb +112 -0
- data/lib/predicator/duration_text.rb +92 -0
- data/lib/predicator/emitter/chains.rb +62 -0
- data/lib/predicator/emitter/statements.rb +86 -0
- data/lib/predicator/emitter.rb +187 -0
- data/lib/predicator/errors.rb +140 -0
- data/lib/predicator/evaluator/access.rb +108 -0
- data/lib/predicator/evaluator/arithmetic.rb +132 -0
- data/lib/predicator/evaluator/calls.rb +129 -0
- data/lib/predicator/evaluator/casts.rb +158 -0
- data/lib/predicator/evaluator/chronology.rb +88 -0
- data/lib/predicator/evaluator/compare.rb +158 -0
- data/lib/predicator/evaluator/json_functions.rb +167 -0
- data/lib/predicator/evaluator/json_scan.rb +303 -0
- data/lib/predicator/evaluator/library.rb +127 -0
- data/lib/predicator/evaluator/math_functions.rb +72 -0
- data/lib/predicator/evaluator/order.rb +120 -0
- data/lib/predicator/evaluator/rich_types.rb +146 -0
- data/lib/predicator/evaluator/statements.rb +151 -0
- data/lib/predicator/evaluator.rb +391 -117
- data/lib/predicator/float_text.rb +94 -0
- data/lib/predicator/iso_text.rb +99 -0
- data/lib/predicator/lexer/literals.rb +115 -0
- data/lib/predicator/lexer/numbers.rb +91 -0
- data/lib/predicator/lexer.rb +194 -0
- data/lib/predicator/parser/durations.rb +108 -0
- data/lib/predicator/parser/operators.rb +86 -0
- data/lib/predicator/parser/postfix.rb +57 -0
- data/lib/predicator/parser/primaries.rb +122 -0
- data/lib/predicator/parser/statements.rb +114 -0
- data/lib/predicator/parser/token_text.rb +52 -0
- data/lib/predicator/parser.rb +217 -438
- data/lib/predicator/result.rb +46 -0
- data/lib/predicator/tagged/reader.rb +139 -0
- data/lib/predicator/tagged/writer.rb +106 -0
- data/lib/predicator/tagged.rb +134 -0
- data/lib/predicator/values.rb +147 -0
- data/lib/predicator/version.rb +4 -1
- data/lib/predicator.rb +384 -14
- metadata +53 -147
- data/.gitignore +0 -11
- data/.rubocop.yml +0 -46
- data/.travis.yml +0 -5
- data/CODE_OF_CONDUCT.md +0 -49
- data/Gemfile +0 -4
- data/HISTORY.md +0 -51
- data/Rakefile +0 -24
- data/lib/predicator/context.rb +0 -18
- data/lib/predicator/lexer.rex +0 -69
- data/lib/predicator/lexer.rex.rb +0 -187
- data/lib/predicator/parser.y +0 -120
- data/lib/predicator/visitors/dot.rb +0 -113
- data/lib/predicator/visitors/each.rb +0 -16
- data/lib/predicator/visitors/instructions.rb +0 -183
- data/lib/predicator/visitors/string.rb +0 -76
- data/lib/predicator/visitors/visitor.rb +0 -100
- data/lib/predicator/visitors.rb +0 -5
- data/predicator.gemspec +0 -34
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Predicator
|
|
4
|
+
# ISO 8601 text read as the `::date` and `::datetime` casts read it: the
|
|
5
|
+
# forms the reference's host parser accepts, which the reference hands such
|
|
6
|
+
# text to.
|
|
7
|
+
#
|
|
8
|
+
# A date is four digits of year, optionally signed (the sign the year's),
|
|
9
|
+
# a month and a day, naming a real calendar date. A datetime is such a
|
|
10
|
+
# date, `T` or a space, hours, minutes and seconds of a time a day has, an
|
|
11
|
+
# optional fraction after a full stop or a comma (read to the microsecond,
|
|
12
|
+
# further digits dropped), and an offset: `Z`, or a sign with two digits of
|
|
13
|
+
# hours and optionally two of minutes, with or without a colon, of at most
|
|
14
|
+
# 23 hours and 59 minutes. The text `-00:00` is refused, as the reference
|
|
15
|
+
# refuses it, while `-0000` and `-00` read as UTC. The offset is applied:
|
|
16
|
+
# the answer is the instant the text names, in UTC, and an instant whose
|
|
17
|
+
# year in UTC falls outside {YEARS} is refused (the reference raises
|
|
18
|
+
# there).
|
|
19
|
+
#
|
|
20
|
+
# Each field of the offset is two digits. The reference's ISA (docs/isa.md,
|
|
21
|
+
# section 5) asks `::datetime` for an ISO 8601 datetime with a UTC offset,
|
|
22
|
+
# and an ISO 8601 offset's hour and minute fields are digits with no sign
|
|
23
|
+
# of their own, so a text such as `+-5:30` is refused here. The
|
|
24
|
+
# reference's host parser reads each field as an integer that may carry a
|
|
25
|
+
# sign and answers an instant for it.
|
|
26
|
+
#
|
|
27
|
+
# The module holds no state. It is private to the package.
|
|
28
|
+
module IsoText
|
|
29
|
+
DATE_TEXT = /\A([+-]?)([0-9]{4})-([0-9]{2})-([0-9]{2})\z/
|
|
30
|
+
DATETIME_TEXT = /\A([+-]?)([0-9]{4})-([0-9]{2})-([0-9]{2})[T ]([0-9]{2}):([0-9]{2}):([0-9]{2})
|
|
31
|
+
(?:[.,]([0-9]+))?(Z|[+-][0-9]{2}(?::?[0-9]{2})?)\z/x
|
|
32
|
+
OFFSET_TEXT = /\A([+-])([0-9]{2}):?([0-9]{2})?\z/
|
|
33
|
+
|
|
34
|
+
# The years an instant may fall in, in UTC.
|
|
35
|
+
YEARS = (-9999..9999)
|
|
36
|
+
|
|
37
|
+
module_function
|
|
38
|
+
|
|
39
|
+
# @param text [String]
|
|
40
|
+
# @return [Date, nil] the date the text names, or nil
|
|
41
|
+
def read_date(text)
|
|
42
|
+
match = DATE_TEXT.match(text)
|
|
43
|
+
civil_date(*match.captures) if match
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param text [String]
|
|
47
|
+
# @return [DateTime, nil] the instant the text names, or nil
|
|
48
|
+
def read_datetime(text)
|
|
49
|
+
instant = read_instant(text)
|
|
50
|
+
in_years(instant) if instant
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# The instant the text names, whatever its year in UTC: a datetime
|
|
54
|
+
# literal in source reads its text so, as the reference's literal does.
|
|
55
|
+
#
|
|
56
|
+
# @param text [String]
|
|
57
|
+
# @return [DateTime, nil] the instant the text names, or nil
|
|
58
|
+
def read_instant(text)
|
|
59
|
+
match = DATETIME_TEXT.match(text)
|
|
60
|
+
return unless match
|
|
61
|
+
|
|
62
|
+
sign, year, month, day, *clock, fraction, zone = match.captures
|
|
63
|
+
local = local_instant(civil_date(sign, year, month, day), clock.map { |part| Integer(part, 10) }, fraction)
|
|
64
|
+
offset = read_offset(zone)
|
|
65
|
+
local.with(epoch_seconds: local.epoch_seconds - offset) if local && offset
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def civil_date(sign, *parts)
|
|
69
|
+
year, month, day = parts.map { |part| Integer(part, 10) }
|
|
70
|
+
year = -year if sign == "-"
|
|
71
|
+
Date.new(year:, month:, day:) if Civil.civil?(year, month, day)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def local_instant(date, (hour, minute, second), fraction)
|
|
75
|
+
return unless date && hour <= 23 && minute <= 59 && second <= 59
|
|
76
|
+
|
|
77
|
+
microsecond = Integer("#{fraction}000000"[0, 6], 10)
|
|
78
|
+
Civil.instant_at(date, hour, minute, second, microsecond:)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def in_years(instant) = (instant if YEARS.cover?(Civil.date_of(instant).year))
|
|
82
|
+
|
|
83
|
+
# The seconds the offset stands east of UTC, or nil when the reference
|
|
84
|
+
# refuses it.
|
|
85
|
+
def read_offset(zone)
|
|
86
|
+
return 0 if zone == "Z"
|
|
87
|
+
return if zone == "-00:00"
|
|
88
|
+
|
|
89
|
+
sign, hours, minutes = OFFSET_TEXT.match(zone).captures
|
|
90
|
+
hours = Integer(hours, 10)
|
|
91
|
+
minutes = Integer(minutes || "0", 10)
|
|
92
|
+
return if hours > 23 || minutes > 59
|
|
93
|
+
|
|
94
|
+
seconds = (hours * 3600) + (minutes * 60)
|
|
95
|
+
sign == "-" ? -seconds : seconds
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
private_constant :IsoText
|
|
99
|
+
end
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Predicator
|
|
4
|
+
module Lexer
|
|
5
|
+
# The string and date literals, read into the scan that includes this
|
|
6
|
+
# module (its scanner, its running line and column, its tokens).
|
|
7
|
+
module Literals
|
|
8
|
+
# The escapes a string literal decodes. Any other escaped character
|
|
9
|
+
# stands for itself, so an unrecognized escape loses its backslash.
|
|
10
|
+
ESCAPES = { "n" => "\n", "t" => "\t", "r" => "\r" }.freeze
|
|
11
|
+
|
|
12
|
+
# A literal's run of characters that neither close it nor escape, by
|
|
13
|
+
# its quote.
|
|
14
|
+
PLAIN = { '"' => /[^"\\]*/, "'" => /[^'\\]*/ }.freeze
|
|
15
|
+
|
|
16
|
+
# The quote a string token carries, by its character.
|
|
17
|
+
QUOTES = { '"' => :double, "'" => :single }.freeze
|
|
18
|
+
|
|
19
|
+
# The refusal of the numeric escape, in either spelling of its letter.
|
|
20
|
+
NUMERIC_ESCAPE = "Unsupported escape sequence \\u in string literal: predicator has no numeric " \
|
|
21
|
+
"escape; write the character itself (string literals are UTF-8)"
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
# A string literal, quoted with `"` or `'`. A raw line feed inside it
|
|
26
|
+
# is content and moves the line; a two-character escape advances the
|
|
27
|
+
# column by two whatever it decodes to, a backslash before a line feed
|
|
28
|
+
# included. A refusal inside the literal spans its opening quote.
|
|
29
|
+
def string
|
|
30
|
+
start = @scanner.pos
|
|
31
|
+
quote = @scanner.getch
|
|
32
|
+
content = read_string(quote)
|
|
33
|
+
return content if content.is_a?(Refusal)
|
|
34
|
+
|
|
35
|
+
text = @scanner.string.byteslice(start...@scanner.pos)
|
|
36
|
+
end_line, end_column = after(text.gsub(/\\./m, ".."), @line, @column)
|
|
37
|
+
emit(:string, content, text.length, quote: QUOTES.fetch(quote),
|
|
38
|
+
end_position: Position.new(line: end_line, column: end_column))
|
|
39
|
+
move(end_line, end_column)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# The literal's content, up to and past its closing quote; or the
|
|
43
|
+
# refusal.
|
|
44
|
+
def read_string(quote)
|
|
45
|
+
content = +""
|
|
46
|
+
loop do
|
|
47
|
+
content << @scanner.scan(PLAIN.fetch(quote))
|
|
48
|
+
char = @scanner.getch
|
|
49
|
+
return content if char == quote
|
|
50
|
+
|
|
51
|
+
piece = char ? escape(quote) : unterminated(quote)
|
|
52
|
+
return piece if piece.is_a?(Refusal)
|
|
53
|
+
|
|
54
|
+
content << piece
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# What the escape after a backslash stands for, or the refusal.
|
|
59
|
+
def escape(quote)
|
|
60
|
+
escaped = @scanner.getch
|
|
61
|
+
return unterminated(quote) if escaped.nil?
|
|
62
|
+
return refuse("unsupported_escape", NUMERIC_ESCAPE, @line, @column + 1) if %w[u U].include?(escaped)
|
|
63
|
+
|
|
64
|
+
ESCAPES.fetch(escaped, escaped)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def unterminated(quote)
|
|
68
|
+
refuse("unterminated_string", "Unterminated #{quote == '"' ? "double" : "single"}-quoted string literal",
|
|
69
|
+
@line, @column + 1)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# A date or datetime literal, `#...#`. A body holding a `T` is read as
|
|
73
|
+
# a datetime and any other as a date, as {IsoText} reads them (the
|
|
74
|
+
# datetime unbounded in its year, as the reference's literal is). A
|
|
75
|
+
# refusal spans the whole literal: through the closing `#`, or through
|
|
76
|
+
# the end of the source when there is none.
|
|
77
|
+
def date
|
|
78
|
+
@scanner.getch
|
|
79
|
+
body = @scanner.scan(/[^#]*/)
|
|
80
|
+
end_line, end_column = after(body, @line, @column + 1)
|
|
81
|
+
return refuse("unterminated_date", "Unterminated date literal", end_line, end_column) unless @scanner.getch
|
|
82
|
+
|
|
83
|
+
type, value = read_date(body)
|
|
84
|
+
return refuse("invalid_#{type}", "Invalid #{type} format: #{body}", end_line, end_column + 1) if value.nil?
|
|
85
|
+
|
|
86
|
+
emit(type, value, body.length + 2)
|
|
87
|
+
move(end_line, end_column + 1)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Moves the running position past a literal that may span lines.
|
|
91
|
+
def move(line, column)
|
|
92
|
+
@line = line
|
|
93
|
+
@column = column
|
|
94
|
+
nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def read_date(body)
|
|
98
|
+
return [:datetime, IsoText.read_instant(body)] if body.include?("T")
|
|
99
|
+
|
|
100
|
+
[:date, IsoText.read_date(body)]
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The line and column just past text that starts at the line and
|
|
104
|
+
# column given: a line feed starts the next line, and every other
|
|
105
|
+
# character, a carriage return included, advances the column by one.
|
|
106
|
+
def after(text, line, column)
|
|
107
|
+
breaks = text.count("\n")
|
|
108
|
+
return [line, column + text.length] if breaks.zero?
|
|
109
|
+
|
|
110
|
+
[line + breaks, text.length - text.rindex("\n")]
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
private_constant :Literals
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Predicator
|
|
4
|
+
module Lexer
|
|
5
|
+
# Numbers and the duration units written after them, read into the scan
|
|
6
|
+
# that includes this module.
|
|
7
|
+
#
|
|
8
|
+
# A number is decimal digits with at most one decimal point, which a
|
|
9
|
+
# digit must follow: `1.` is the number 1 and then a dot. There is no
|
|
10
|
+
# sign and no exponent. A number followed by a duration unit is an
|
|
11
|
+
# integer token, or a fractional number token where it has a point, then
|
|
12
|
+
# one token per unit; a number followed by anything else is an integer
|
|
13
|
+
# or a float token, so `1.5x` is a float and then an identifier.
|
|
14
|
+
#
|
|
15
|
+
# The units are read one after another, the two-letter spellings (`ms`,
|
|
16
|
+
# `mo`) before the one-letter ones, and the read stops at the first
|
|
17
|
+
# character that is not a unit: `5days` is 5, the unit `d`, and the
|
|
18
|
+
# identifier `ays`.
|
|
19
|
+
#
|
|
20
|
+
# After the FIRST unit the reference's lexer goes on from a copy of the
|
|
21
|
+
# text that starts just after that unit and ends at the first line feed
|
|
22
|
+
# past the copy's first character. Any further units are read inside
|
|
23
|
+
# that copy, and the source from that line feed on is not read. The
|
|
24
|
+
# column advances by the grapheme clusters, not the characters, between
|
|
25
|
+
# the end of the number and the start of what the copy holds after the
|
|
26
|
+
# units, the dropped text included. Both are reproduced here, the corpus
|
|
27
|
+
# fixing neither: `5d x`, a line feed and `y` scans as 5, `d` and `x` at
|
|
28
|
+
# column 6, then the end of input; `5dh`, a line feed and `x` scans as
|
|
29
|
+
# 5, `d`, `h` and the end of input at column 6 of line 1.
|
|
30
|
+
module Numbers
|
|
31
|
+
# The units a duration names, the two-letter spellings first.
|
|
32
|
+
UNIT = /m[so]|[ydhmsw]/
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
# A decimal too large for a finite Float reads as an infinite one, and
|
|
37
|
+
# is read without asking the host for it ({FloatText.read}): the
|
|
38
|
+
# reference's lexer raises there, and the emitter refuses the literal.
|
|
39
|
+
def number
|
|
40
|
+
text = @scanner.scan(/[0-9]+(?:\.[0-9]+)?/)
|
|
41
|
+
whole, fraction = text.split(".")
|
|
42
|
+
unless @scanner.match?(UNIT)
|
|
43
|
+
value = fraction ? FloatText.read(text) || Float::INFINITY : Integer(whole, 10)
|
|
44
|
+
return emit(fraction ? :float : :integer, value, text.length)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
leading = fraction ? Fraction.new(whole: Integer(whole, 10), fraction:) : Integer(whole, 10)
|
|
48
|
+
emit(fraction ? :fractional_number : :integer, leading, text.length)
|
|
49
|
+
units
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def units
|
|
53
|
+
text = +""
|
|
54
|
+
dropped = nil
|
|
55
|
+
while (unit = @scanner.scan(UNIT))
|
|
56
|
+
@tokens << Token.new(type: :duration_unit, line: @line, column: @column + text.length,
|
|
57
|
+
length: unit.length, value: unit)
|
|
58
|
+
text << unit
|
|
59
|
+
dropped ||= keep_line
|
|
60
|
+
end
|
|
61
|
+
@column += consumed(text, dropped)
|
|
62
|
+
nil
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# After the first unit: goes on scanning over the text from here to the
|
|
66
|
+
# first line feed past its first character alone, and answers the text
|
|
67
|
+
# dropped after it ("" when the kept text runs to the end of the source).
|
|
68
|
+
def keep_line
|
|
69
|
+
line = @scanner.check(/.[^\n]*/m) || ""
|
|
70
|
+
return "" if @scanner.pos + line.bytesize == @scanner.string.bytesize
|
|
71
|
+
|
|
72
|
+
dropped = @scanner.string.byteslice((@scanner.pos + line.bytesize)..)
|
|
73
|
+
@scanner = StringScanner.new(line)
|
|
74
|
+
@chain_end = 0
|
|
75
|
+
dropped
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The grapheme clusters the units and the dropped text spanned, as the
|
|
79
|
+
# reference counts them. The units are ASCII letters, so only the
|
|
80
|
+
# cluster just after them can join the last; a carriage return ending
|
|
81
|
+
# the kept text joins the dropped line feed.
|
|
82
|
+
def consumed(unit_text, dropped)
|
|
83
|
+
count = "#{unit_text}#{@scanner.check(/\X/)}".scan(/\X/).size - (@scanner.eos? ? 0 : 1)
|
|
84
|
+
return count if dropped.empty?
|
|
85
|
+
|
|
86
|
+
count + dropped.scan(/\X/).size - (!@scanner.eos? && @scanner.string.end_with?("\r") ? 1 : 0)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
private_constant :Numbers
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "strscan"
|
|
4
|
+
require_relative "values"
|
|
5
|
+
require_relative "errors"
|
|
6
|
+
require_relative "iso_text"
|
|
7
|
+
require_relative "float_text"
|
|
8
|
+
require_relative "lexer/literals"
|
|
9
|
+
require_relative "lexer/numbers"
|
|
10
|
+
|
|
11
|
+
module Predicator
|
|
12
|
+
# The compiler's scanner: source text in, a token stream out, or the one
|
|
13
|
+
# refusal that stopped it. It reads the source with a StringScanner and
|
|
14
|
+
# answers the tokens the reference's lexer answers, at the positions it
|
|
15
|
+
# gives them; there is no partial stream and no recovery.
|
|
16
|
+
#
|
|
17
|
+
# A position counts lines and columns from 1, and a column counts
|
|
18
|
+
# characters (Unicode codepoints, as the reference counts them), never
|
|
19
|
+
# bytes. A space and a tab advance the column by one, a line feed starts
|
|
20
|
+
# the next line, and a carriage return outside a literal advances nothing.
|
|
21
|
+
# One count is the reference's own and not a column: the stretch a run of
|
|
22
|
+
# duration units consumes is counted in grapheme clusters ({Numbers}).
|
|
23
|
+
#
|
|
24
|
+
# The source is a String in UTF-8 whose encoding is valid. The module holds
|
|
25
|
+
# no state; each call scans with its own {Scan}. It is private to the
|
|
26
|
+
# package.
|
|
27
|
+
module Lexer
|
|
28
|
+
# One token. The type is one of the reference's token names, as a
|
|
29
|
+
# Symbol; the length counts the characters of source the token covers.
|
|
30
|
+
# A string token also carries its quote (`:double` or `:single`) and
|
|
31
|
+
# the exclusive end of the literal, which is on a later line than its
|
|
32
|
+
# start when the literal holds a raw line feed. Every other token
|
|
33
|
+
# carries neither.
|
|
34
|
+
#
|
|
35
|
+
# @!attribute [r] value
|
|
36
|
+
# @return [Object] what the token stands for: an identifier's or a
|
|
37
|
+
# word's text, an Integer, a Float, a {Fraction}, a String's
|
|
38
|
+
# content, true or false, {Predicator::UNDEFINED}, nil, a
|
|
39
|
+
# {Predicator::Date}, a {Predicator::DateTime}, a unit's or an
|
|
40
|
+
# operator's text; nil for the end of input
|
|
41
|
+
Token = Data.define(:type, :line, :column, :length, :value, :quote, :end_position) do
|
|
42
|
+
def initialize(type:, line:, column:, length:, value: nil, quote: nil, end_position: nil) = super
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The whole number and the fraction digits of a decimal number written
|
|
46
|
+
# before a duration unit (`0.5h`). The fraction stays the digits written,
|
|
47
|
+
# so a duration is expanded from exact digits rather than from a Float.
|
|
48
|
+
Fraction = Data.define(:whole, :fraction)
|
|
49
|
+
|
|
50
|
+
# The extent of a refusal: from its start, inclusive, to its end,
|
|
51
|
+
# exclusive, each a {Predicator::Position}.
|
|
52
|
+
Span = Data.define(:start, :end)
|
|
53
|
+
|
|
54
|
+
# Why the source does not scan: a {Predicator::ParseError} (whose
|
|
55
|
+
# position is the span's start) and its {Span}.
|
|
56
|
+
Refusal = Data.define(:error, :span)
|
|
57
|
+
|
|
58
|
+
# The words that are not identifiers, by their exact spelling, and the
|
|
59
|
+
# token type and value each one becomes.
|
|
60
|
+
RESERVED = {
|
|
61
|
+
"true" => [:boolean, true], "false" => [:boolean, false],
|
|
62
|
+
"undefined" => [:undefined, UNDEFINED], "null" => [:null, nil],
|
|
63
|
+
"AND" => [:and_op, "AND"], "OR" => [:or_op, "OR"], "NOT" => [:not_op, "NOT"],
|
|
64
|
+
"and" => [:and_op, "and"], "or" => [:or_op, "or"], "not" => [:not_op, "not"],
|
|
65
|
+
"IN" => [:in_op, "IN"], "in" => [:in_op, "in"],
|
|
66
|
+
"CONTAINS" => [:contains_op, "CONTAINS"], "contains" => [:contains_op, "contains"],
|
|
67
|
+
"if" => [:if_kw, "if"], "else" => [:else_kw, "else"], "while" => [:while_kw, "while"],
|
|
68
|
+
"ago" => [:ago_op, "ago"], "from" => [:from_op, "from"], "now" => [:now_op, "now"],
|
|
69
|
+
"next" => [:next_op, "next"], "last" => [:last_op, "last"]
|
|
70
|
+
}.freeze
|
|
71
|
+
|
|
72
|
+
# The punctuation, by spelling. A single `&` or `|` is not a token.
|
|
73
|
+
OPERATORS = {
|
|
74
|
+
"===" => :strict_equal, "!==" => :strict_ne, ">=" => :gte, "<=" => :lte, "==" => :equal_equal,
|
|
75
|
+
"!=" => :ne, "&&" => :and_and, "||" => :or_or, "::" => :double_colon, ">" => :gt, "<" => :lt,
|
|
76
|
+
"=" => :eq, "!" => :bang, "+" => :plus, "-" => :minus, "*" => :multiply, "/" => :divide,
|
|
77
|
+
"%" => :modulo, "(" => :lparen, ")" => :rparen, "[" => :lbracket, "]" => :rbracket,
|
|
78
|
+
"{" => :lbrace, "}" => :rbrace, ":" => :colon, "," => :comma, ";" => :semicolon, "." => :dot
|
|
79
|
+
}.freeze
|
|
80
|
+
|
|
81
|
+
# Every operator, the longest spelling tried first.
|
|
82
|
+
OPERATOR = Regexp.union(OPERATORS.keys.sort_by { |text| -text.length })
|
|
83
|
+
|
|
84
|
+
# Scans source text.
|
|
85
|
+
#
|
|
86
|
+
# @param source [String] UTF-8 text
|
|
87
|
+
# @return [Array<Token>, Refusal] every token, the last of type `:eof`,
|
|
88
|
+
# or the refusal that stopped the scan
|
|
89
|
+
def self.tokenize(source) = Scan.new(source).run
|
|
90
|
+
|
|
91
|
+
# One scan of one source: the scanner, the running line and column, and
|
|
92
|
+
# the tokens so far. Each call to {Lexer.tokenize} makes its own.
|
|
93
|
+
class Scan
|
|
94
|
+
include Literals
|
|
95
|
+
include Numbers
|
|
96
|
+
|
|
97
|
+
# What a character starts, tried in order; anything else is
|
|
98
|
+
# punctuation or a refusal.
|
|
99
|
+
STARTS = [[/[ \t\r\n]/, :blank], [/[0-9]/, :number], [/[A-Za-z_]/, :word], [/["']/, :string],
|
|
100
|
+
[/#/, :date]].freeze
|
|
101
|
+
|
|
102
|
+
def initialize(source)
|
|
103
|
+
@scanner = StringScanner.new(source)
|
|
104
|
+
@line = 1
|
|
105
|
+
@column = 1
|
|
106
|
+
@tokens = []
|
|
107
|
+
@chain_end = 0
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# @return [Array<Token>, Refusal]
|
|
111
|
+
def run
|
|
112
|
+
until @scanner.eos?
|
|
113
|
+
start = STARTS.find { |pattern, _| @scanner.match?(pattern) }
|
|
114
|
+
refusal = start ? send(start.last) : operator
|
|
115
|
+
return refusal if refusal
|
|
116
|
+
end
|
|
117
|
+
@tokens << Token.new(type: :eof, line: @line, column: @column, length: 0)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
private
|
|
121
|
+
|
|
122
|
+
def blank
|
|
123
|
+
@scanner.scan(/[ \t\r\n]+/).each_char do |char|
|
|
124
|
+
if char == "\n"
|
|
125
|
+
@line += 1
|
|
126
|
+
@column = 1
|
|
127
|
+
elsif char != "\r"
|
|
128
|
+
@column += 1
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
nil
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# An identifier, a reserved word, or a function's name. A dotted name
|
|
135
|
+
# is one token only where a call follows it; anywhere else its first
|
|
136
|
+
# part stands alone and the dot is the next token. Every later part of
|
|
137
|
+
# a dotted name no call followed ends where the first did, so no call
|
|
138
|
+
# follows it either, and the name is not read again from each part.
|
|
139
|
+
def word
|
|
140
|
+
start = @scanner.pos
|
|
141
|
+
name = @scanner.scan(/[A-Za-z_][A-Za-z0-9_]*/)
|
|
142
|
+
return dotted(name) if start >= @chain_end && @scanner.match?(/\.[A-Za-z_]/)
|
|
143
|
+
|
|
144
|
+
type, value = RESERVED.fetch(name) { [:identifier, name] }
|
|
145
|
+
type = :function_name if type == :identifier && call_follows?(@scanner.pos)
|
|
146
|
+
emit(type, value, name.length)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def dotted(name)
|
|
150
|
+
chain = @scanner.check(/(?:\.[A-Za-z_][A-Za-z0-9_]*)+/)
|
|
151
|
+
after = @scanner.pos + chain.bytesize
|
|
152
|
+
if call_follows?(after)
|
|
153
|
+
@scanner.pos = after
|
|
154
|
+
return emit(:qualified_function_name, name + chain, name.length + chain.length)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
@chain_end = after
|
|
158
|
+
emit(*RESERVED.fetch(name) { [:identifier, name] }, name.length)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Whether an opening parenthesis follows the byte offset, past any
|
|
162
|
+
# whitespace. A lookahead only: the scanner is left where it was.
|
|
163
|
+
def call_follows?(offset)
|
|
164
|
+
at = @scanner.pos
|
|
165
|
+
@scanner.pos = offset
|
|
166
|
+
@scanner.match?(/[ \t\r\n]*\(/)
|
|
167
|
+
ensure
|
|
168
|
+
@scanner.pos = at
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def operator
|
|
172
|
+
text = @scanner.scan(OPERATOR)
|
|
173
|
+
return emit(OPERATORS.fetch(text), text, text.length) if text
|
|
174
|
+
|
|
175
|
+
refuse("unexpected_character", "Unexpected character '#{@scanner.check(/./m)}'", @line, @column + 1)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def emit(type, value, length, **literal)
|
|
179
|
+
@tokens << Token.new(type:, line: @line, column: @column, length:, value:, **literal)
|
|
180
|
+
@column += length
|
|
181
|
+
nil
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# A refusal at the running position, spanning to the end given.
|
|
185
|
+
def refuse(reason, message, end_line, end_column)
|
|
186
|
+
position = Position.new(line: @line, column: @column)
|
|
187
|
+
Refusal.new(error: ParseError.new(reason:, message:, position:),
|
|
188
|
+
span: Span.new(start: position, end: Position.new(line: end_line, column: end_column)))
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
private_constant :Scan
|
|
192
|
+
end
|
|
193
|
+
private_constant :Lexer
|
|
194
|
+
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Predicator
|
|
4
|
+
module Parser
|
|
5
|
+
# Duration literals and the words that make one a relative date, read
|
|
6
|
+
# into the parse that includes this module.
|
|
7
|
+
#
|
|
8
|
+
# The lexer writes `3d8h` as a number and a unit per component. A literal
|
|
9
|
+
# written in whole numbers keeps its components as written, a unit
|
|
10
|
+
# written twice included. A component written with a point is expanded
|
|
11
|
+
# as the reference expands it: the whole part keeps its unit when it is
|
|
12
|
+
# not zero, and the fraction's worth, which must be a whole number of
|
|
13
|
+
# milliseconds (a month 30 days, a year 365), is spent largest first
|
|
14
|
+
# through days, hours, minutes, seconds and milliseconds, the pair
|
|
15
|
+
# `[0, unit]` standing for a component worth nothing. An inexact
|
|
16
|
+
# fraction is refused over its component; a unit named twice after an
|
|
17
|
+
# expansion is refused over the whole literal, the alphabetically first
|
|
18
|
+
# such unit named.
|
|
19
|
+
module Durations
|
|
20
|
+
# Each unit's suffix and its worth in milliseconds.
|
|
21
|
+
MILLISECONDS = DurationText::UNITS.values.to_h.freeze
|
|
22
|
+
|
|
23
|
+
# The units a fraction's worth is spent into, largest first.
|
|
24
|
+
LADDER = %w[d h m s ms].freeze
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
# The duration (or relative date) a number starts, when a unit follows
|
|
29
|
+
# it; nil when none does, and nothing is consumed.
|
|
30
|
+
def duration_from(first, start)
|
|
31
|
+
return unless at?(:duration_unit)
|
|
32
|
+
|
|
33
|
+
unit = advance
|
|
34
|
+
components = [[first, unit.value, between(start, end_of(unit))]]
|
|
35
|
+
components << component while component_follows?
|
|
36
|
+
units = whole_units(components)
|
|
37
|
+
direction(Ast::Duration.new(units:, position: start, span: between(start, end_of(previous))))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def component_follows?
|
|
41
|
+
%i[integer fractional_number].include?(peek.type) && @tokens[@index + 1].type == :duration_unit
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def component
|
|
45
|
+
number = advance
|
|
46
|
+
unit = advance
|
|
47
|
+
[number.value, unit.value, between(start_of(number), end_of(unit))]
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def whole_units(components)
|
|
51
|
+
written = components.map { |value, unit, _| [value, unit] }
|
|
52
|
+
return written if written.none? { |value, _| fraction?(value) }
|
|
53
|
+
|
|
54
|
+
units = components.flat_map { |parts| expanded(*parts) }
|
|
55
|
+
twice = repeated(units)
|
|
56
|
+
twice ? unit_twice(twice, components) : units
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The alphabetically first unit named more than once, if one is.
|
|
60
|
+
def repeated(units) = units.map(&:last).tally.select { |_, count| count > 1 }.keys.min
|
|
61
|
+
|
|
62
|
+
def unit_twice(unit, components)
|
|
63
|
+
refuse("duration_unit_twice", "Duration literal names the '#{unit}' unit twice after expanding a fraction",
|
|
64
|
+
nil, between(components.first.last.start, components.last.last.end))
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def fraction?(value) = value.is_a?(Lexer::Fraction)
|
|
68
|
+
|
|
69
|
+
def expanded(value, unit, span)
|
|
70
|
+
return [[value, unit]] unless fraction?(value)
|
|
71
|
+
|
|
72
|
+
pairs = fraction_units(value, unit)
|
|
73
|
+
return pairs if pairs
|
|
74
|
+
|
|
75
|
+
refuse("duration_fraction", "Duration fraction is not a whole number of milliseconds: " \
|
|
76
|
+
"#{value.whole}.#{value.fraction}#{unit}", nil, span)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# The whole-unit pairs one component with a point is worth, or nil
|
|
80
|
+
# when its fraction is not a whole number of milliseconds.
|
|
81
|
+
def fraction_units(value, unit)
|
|
82
|
+
remainder, inexact = (Integer(value.fraction, 10) * MILLISECONDS.fetch(unit)).divmod(10**value.fraction.size)
|
|
83
|
+
return unless inexact.zero?
|
|
84
|
+
|
|
85
|
+
pairs = (value.whole.positive? ? [[value.whole, unit]] : []) + spent(remainder)
|
|
86
|
+
pairs.empty? ? [[0, unit]] : pairs
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def spent(milliseconds)
|
|
90
|
+
LADDER.filter_map do |suffix|
|
|
91
|
+
amount, milliseconds = milliseconds.divmod(MILLISECONDS.fetch(suffix))
|
|
92
|
+
[amount, suffix] if amount.positive?
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# `ago`, or `from now`, after a duration; the node is at the word.
|
|
97
|
+
def direction(duration)
|
|
98
|
+
return duration unless %i[ago_op from_op].include?(peek.type)
|
|
99
|
+
|
|
100
|
+
word = advance
|
|
101
|
+
last = word.type == :ago_op ? word : expect(:now_op, "expected_now", "Expected 'now' after 'from' but found")
|
|
102
|
+
Ast::RelativeDate.new(duration:, direction: word.type == :ago_op ? :ago : :future, position: start_of(word),
|
|
103
|
+
span: reaching(duration, last))
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
private_constant :Durations
|
|
107
|
+
end
|
|
108
|
+
end
|