acute 1.0.0

data/lib/acute/grammar/words.rb

@@ -0,0 +1,307 @@
+module Acute
+  # Atoms led by a word: weekdays, months, parts of day, next/last, early/late, "in", "a week"
+  class Grammar
+    private
+
+    # An if chain, the commonest kinds first: a case over constants tests each with a method call
+    def atom(loose)
+      t = @t
+      i = @i
+      k = t[i]
+      v = t[i + 1]
+      if k == L::NUM then number_atom(loose)
+      elsif k == L::CLOCK then written(v, t[i + 2])
+      elsif k == L::DAY
+        @i += S
+        # "tomorrow week": a week after it
+        if t[i + S] == L::UNIT && WEEKS[u = t[i + S + 1]] && (k = t[i + 2 * S]) != L::NUM && k != L::ORD
+          @i += S
+          return Shift.new(1, u, DayRef.new(v))
+        end
+        DayRef.new(v)
+      elsif k == L::WDAY
+        return failure(:incomplete) if t[i + 2] & L::PLURAL != 0
+        @i += S
+        around(of(Weekday.new(v)))
+      elsif k == L::MONTH then around(month_atom)
+      elsif k == L::REL then rel_atom
+      elsif k == L::NAMED
+        @i += S
+        named(v)
+      elsif k == L::ORD then ordinal_atom
+      elsif k == L::PART
+        return failure(:incomplete) if t[i + 2] & L::PLURAL != 0
+        @i += S
+        of(Part.new(v))
+      elsif k == L::NOW
+        @i += S
+        NOW
+      elsif k == L::HOLIDAY
+        return failure(:incomplete) if t[i + 2] & L::PLURAL != 0
+        @i += S
+        # D48: "labor day weekend" is the long weekend with the holiday
+        if t[@i] == L::WEEKEND
+          @i += S
+          return Within.new(WEEKEND, Holiday.new(v))
+        end
+        around(of(Holiday.new(v)))
+      elsif k == L::WORD then word_atom(v)
+      elsif k == L::ART then article_atom
+      elsif k == L::FUZZ then fuzzy_atom
+      elsif k == L::WEEKEND
+        @i += S
+        around(WEEKEND)
+      elsif k == L::SEASON
+        @i += S
+        of(Season.new(v))
+      elsif k == L::UNIT
+        return clock_words(15) if v == :quarter && @t[i + S] == L::REL || @t[i + S + 1] == :to
+        return iso_week if v == :week && week_number?(i + S)
+        # D44: "H1 2027", "H2 2026"
+        if v == :hour && t[i + 4] - t[i + 3] == 1 && t[i + S] == L::NUM && t[i + S + 1].between?(1, 2) && t[i + 4] == t[i + S + 3] &&
+           t[i + 2 * S] == L::NUM && t[i + 2 * S + 2] & L::DIGITS == 4
+          @i += 3 * S
+          return Within.new(half(t[i + S + 1]), Date.new(t[i + 2 * S + 1], nil, nil, nil, false))
+        end
+        # "the year 1980", "in the year '89", "the year twenty ten", "the year two thousand"
+        if v == :year && t[i + S] == L::NUM && ((f = t[i + S + 2]) & L::DIGITS == 4 || f & L::SHORT != 0 ||
+                                                (f & L::WORDY != 0 && (t[i + 2 * S] == L::NUM || t[i + 2 * S + 1] == :thousand)))
+          @i += S
+          return number_atom(true)
+        end
+        # D48: "year 247": three digits after "year" are the year as written
+        if v == :year && t[i + S] == L::NUM && t[i + S + 2] & L::DIGITS == 3 && t[i + S + 2] & L::ZERO == 0 && i + 2 * S >= @n
+          @i += 2 * S
+          return Date.new(t[i + S + 1], nil, nil, nil, false)
+        end
+        if (w = t[i + S + 1]) == :after || w == :before # "the day after tomorrow"
+          @i += S
+          # D44: "the week after next", "the month after the next", "week before last": the one after next
+          # week or before last week, when "next" or "last" has nothing of its own
+          j = i + 2 * S
+          j += S if t[j] == L::WORD && t[j + 1] == :the
+          if t[j] == L::REL && (r = t[j + 1]) == (w == :after ? :next : :last) && nothing_own?(j + S)
+            @i = j + S
+            return Shift.new(w == :after ? 1 : -1, v, Rel.new(r, Unit.new(v, nil)))
+          end
+          return shift_tail(Unit.new(v, 1))
+        end
+        return shift_tail(Unit.new(v, 1)) if starting?(i + S) && (@i += S) # "the week starting february 4"
+        # D44: "the week of october 6th", "the month of may": the one that holds it. D51: "the year of 2008"
+        if (v == :week || v == :month || v == :year) && w == :of && t[i + S] == L::WORD
+          @i += 2 * S
+          node = item(true)
+          return Failure === node ? node : Holding.new(v, node)
+        end
+        failure(:incomplete)
+      elsif k == L::DECADE then decade_atom
+      elsif k == L::MER && t[i + S] == L::CLOCK && i >= S && t[i - S] == L::NUM then meridiem_first(v)
+      else failure(:conflict)
+      end
+    end
+
+    # D48: "2020/10/9 PM 2:26": a meridiem before the clock after a date, the East Asian order
+    def meridiem_first(v)
+      at = @i
+      @i += S
+      c = written(@t[@i + 1], @t[@i + 2])
+      return failure(:conflict, at) unless Clock === c && !c.meridiem && c.hour >= 1 && c.hour <= 12
+      Clock.new(c.hour, c.min, c.sec, v == 0 ? :am : :pm, nil, false, c.seconds, c.frac)
+    end
+
+    # "the 90s", "1990s"; "30s ago" is seconds
+    def decade_atom
+      return shift_tail(quantity) if SHIFT_TAIL[@t[@i + S + 1]] # D47: "5400s ago" as "30s ago"
+      v = @t[@i + 1]
+      @i += S
+      Years.new(:decade, v, v < 100)
+    end
+
+    def word_atom(v)
+      i = @i
+      case v
+      when :in
+        @i += S
+        if @t[@i + 1] == :around # D14: "in about 5 hours"
+          @approx = true
+          @i += S
+        end
+        if (q = duration_quantity)
+          return q if Failure === q
+          return shifts(1, q, more_units(q), nil)
+        end
+        @i += S if @t[@i + 1] == :the
+        return failure(:incomplete) if @i >= @n
+        # D42: "in 3" is a count without its unit, not 3 o'clock
+        return failure(:incomplete) if @t[@i] == L::NUM && @t[@i + 2] & L::DIGITS <= 2 && @i + S >= @n
+        atom(false)
+      when :within
+        @i += S
+        if @t[@i + 1] == :around
+          @approx = true
+          @i += S
+        end
+        # "within the next 5 days" is within 5 days
+        @i += 2 * S if @t[@i + 1] == :the && @t[@i + S] == L::REL && @t[@i + S + 1] == :next
+        @i += S if @t[@i] == L::REL && @t[@i + 1] == :next && @t[@i + S] == L::NUM # "within next 10 months"
+        q = quantity
+        q && !(Failure === q) ? Rel.new(:next, q) : failure(:incomplete, i) # D31
+      when :q
+        return failure(:incomplete) unless @t[i + S] == L::NUM && (1..4).cover?(@t[i + S + 1])
+        @i += 2 * S
+        Nth.new(@t[i + S + 1], Unit.new(:quarter, nil))
+      when :fy
+        return failure(:incomplete) unless @t[i + S] == L::NUM
+        @i += 2 * S
+        y = @t[i + S + 1]
+        Fiscal.new(@t[i + S + 2] & L::DIGITS == 2 ? 2000 + y : y)
+      when :half
+        (q = quantity) ? shift_tail(q) : clock_words(30)
+      when :sametime then same_time
+      when :closest then closest(1)
+      when :all # D48: "all week" is this whole week
+        return failure(:conflict) unless @t[i + S] == L::UNIT && @t[i + S + 2] & L::PLURAL == 0
+        @i += S
+        this_unit
+      when :rest # D48: "rest of the week" is from now to its end, as "through this week" (D16)
+        return failure(:conflict) unless @t[i + S] == L::WORD && @t[i + S + 1] == :of
+        @i += 2 * S
+        @i += S if @t[@i + 1] == :the
+        return failure(:incomplete) if @i >= @n
+        node = @t[@i] == L::UNIT ? this_unit : atom(true)
+        Failure === node ? node : Open.new(:until, node)
+      when :hundred, :thousand, :million then (q = quantity) ? shift_tail(q) : failure(:conflict)
+      when :couple
+        (q = quantity) ? shift_tail(q) : failure(@i + S < @n ? :conflict : :incomplete, @i + S)
+      else failure(:conflict)
+      end
+    end
+
+    # D48: "the closest monday to oct 5th", "the 3rd closest xmas to today"
+    def closest(n)
+      @i += S
+      node = atom(true)
+      return node if Failure === node
+      return failure(@i < @n ? :conflict : :incomplete) unless @t[@i + 1] == :to && @t[@i] == L::WORD
+      @i += S
+      anchor = atom(true)
+      Failure === anchor ? anchor : Closest.new(n, node, anchor)
+    end
+
+    def article_atom
+      i = @i
+      if @t[i + S] == L::UNIT && ((u = @t[i + S + 1]) == :quarter || u == :minute) && clock_word?(i + 2 * S) && hour?(i + 3 * S)
+        return clock_words(u == :quarter ? 15 : 1)
+      end
+      q = quantity
+      return failure(:conflict) unless q
+      shift_tail(q)
+    end
+
+    def rel_atom
+      w = @t[@i + 1]
+      start = @i
+      @i += S
+      if w == :this && @t[@i] == L::REL && @t[@i + 1] != :this # D40: "this coming week"
+        w = @t[@i + 1] == :next ? :coming : :past # D44: "this past sunday" is "past sunday"; D48: so is "this last"
+        @i += S
+      end
+      k = @t[@i]
+      v = @t[@i + 1]
+      # D42: "this second", "last second": with nothing after it to count, "second" is the unit
+      if k == L::ORD && unit_at(@i) && ![L::WDAY, L::UNIT, L::WEEKEND].include?(@t[@i + S])
+        k = L::UNIT
+        v = :second
+      end
+      return failure(:conflict) if (k == L::WDAY || k == L::PART) && @t[@i + 2] & L::PLURAL != 0
+      if k == L::WDAY || k == L::UNIT || k == L::WEEKEND
+        node = k == L::WDAY ? Weekday.new(v) : k == L::UNIT ? Unit.new(v, nil) : WEEKEND
+        @i += S
+        # "last friday of june"; "last sunday in march" when a month follows
+        return of(Nth.new(-1, node)) if w == :last && (@t[@i + 1] == :of || (@t[@i + 1] == :in && @t[@i + S] == L::MONTH))
+        # D48: "last tuesday this month" is the last of that month
+        if w == :last && k == L::WDAY && @t[@i] == L::REL && @t[@i + S + 1] == :month && @t[@i + S] == L::UNIT
+          month = atom(true)
+          return Within.new(Nth.new(-1, node), month)
+        end
+        Rel.new(w, node)
+      elsif k == L::NUM && unit_at(@i + S) && w != :this # D50: "this" takes no count: "this 5 years ago" is "5 years ago"
+        q = quantity
+        Rel.new(w, q)
+      elsif k == L::MONTH && @t[@i + S] != L::NUM && @t[@i + S] != L::ORD
+        @i += S
+        Rel.new(w, Date.new(nil, v, nil, nil, false))
+      elsif k == L::NUM && @t[@i + S] == L::SLASH && @t[@i + 3 * S] != L::SLASH && @t[@i + 2 * S] == L::NUM
+        d = atom(true) # D48: "last 9/11": the latest one, as "last christmas"
+        Failure === d || !(Date === d) ? failure(:conflict, start) : Rel.new(w, d)
+      elsif k == L::WORD && v == :work_week
+        @i += S # D48: "next working week" is Monday to Friday of next week
+        Within.new(Range.new(Weekday.new(1), Weekday.new(5)), Rel.new(w, Unit.new(:week, nil)))
+      elsif k == L::SEASON || k == L::HOLIDAY || k == L::PART
+        @i += S
+        Rel.new(w, k == L::SEASON ? Season.new(v) : k == L::HOLIDAY ? Holiday.new(v) : Part.new(v))
+      elsif k == L::WORD && v == :fy && @t[@i + S] != L::NUM # D17
+        @i += S
+        Rel.new(w, Fiscal.new(nil))
+      elsif (k == L::WORD && v == :q) || (k == L::ORD && @t[@i + S + 1] == :quarter) # D32: "next q1", "last 2nd quarter"
+        q = atom(false)
+        Failure === q ? q : Rel.new(w, q)
+      elsif (k == L::NUM && (@t[@i + S] == L::MER || @t[@i + S] == L::NAMED)) || k == L::CLOCK || k == L::NAMED
+        c = k == L::NAMED ? (@i += S) && named(v) : k == L::CLOCK ? written(v, @t[@i + 2]) : number(v, @t[@i + 2], true)
+        Failure === c ? c : Rel.new(w, c) # D18: "next 3pm"
+      else
+        failure(@i < @n ? :conflict : :incomplete, @i < @n ? @i : start)
+      end
+    end
+
+    EDGES = { eod: %i[end day], eom: %i[end month], eoy: %i[end year], bom: %i[start month], boy: %i[start year] }.freeze
+
+    def day_named?(i)
+      k = @t[i]
+      k == L::DAY || k == L::WDAY || k == L::MONTH
+    end
+
+    # "early next week", "mid-october", "end of the month", "late tonight"
+    def fuzzy_atom
+      pos = @t[@i + 1]
+      @i += S
+      # "mid day", "mid-night": midday and midnight
+      if pos == :mid && ((day = @t[@i + 1] == :day) || @t[@i + 1] == :night) && (@t[@i] == L::UNIT || @t[@i] == L::PART)
+        @i += S
+        return named(day ? 12 : 0)
+      end
+      # D2: "EOD", "end of day tomorrow", "EOD friday": the end of the day named next, else of today; "EOM",
+      # "BOY": of this month or year
+      if (edge = EDGES[pos])
+        pos, unit = edge
+      elsif @t[@i + 1] == :of && @t[@i + S] == L::UNIT && @t[@i + S + 1] == :day && day_named?(@i + 2 * S)
+        @i += 2 * S
+        unit = :day
+      end
+      if unit
+        node = unit == :day && day_named?(@i) ? atom(true) : Rel.new(:this, Unit.new(unit, nil))
+        return Failure === node ? node : Fuzzy.new(pos, node)
+      end
+      # "early tomorrow evening", "late friday night": the part of that day
+      if (@t[@i] == L::DAY || @t[@i] == L::WDAY) && @t[@i + S] == L::PART
+        day = atom(true)
+        return day if Failure === day
+        part = atom(true)
+        return Failure === part ? part : Within.new(Fuzzy.new(pos, part), day)
+      end
+      @i += S if @t[@i + 1] == :of
+      if @t[@i + 1] == :around && @t[@i] == L::WORD # D48: "beginning of around next week", as "around the beginning of"
+        @approx = true
+        @i += S
+      end
+      @i += S if @t[@i + 1] == :the
+      @i += S if @t[@i] == L::DASH && (@t[@i + S] == L::NUM || @t[@i + S] == L::DECADE) # "mid-1990s"
+      return failure(:incomplete) if @i >= @n
+      node = @t[@i] == L::UNIT ? this_unit : atom(true)
+      return node if Failure === node
+      # D44: "end of past week" is the end of last week; a rolling week's edges are now and a week ago
+      node = Rel.new(:last, node.node) if Rel === node && node.word == :past && Unit === node.node && !node.node.count
+      Fuzzy.new(pos, node)
+    end
+  end
+end

data/lib/acute/grammar.rb

@@ -0,0 +1,378 @@
+require_relative "lexer"
+require_relative "calendar"
+
+module Acute
+  # Tokens to a tree of what was said, by recursive descent. It knows nothing of now, the zone or the
+  # D-rules: a bare "4" stays hour 4 with no am/pm, and "10/11" stays two numbers and a slash; resolve
+  # decides what they mean.
+  #
+  #   phrase := (after|before|since|until|by|starting) body | around body | body
+  #   body   := between item and item | from item to item | item [(to|through|until|-) item]
+  #             | atom+ (between item and | from item to) item
+  #   item   := atom+ [zone]        each atom is one constituent: a date, a weekday, a clock, a shift...
+  #
+  # "around" anywhere in an item makes the whole phrase Approx (D14).
+  #
+  # An item's atoms are nested by size, the coarsest outside: "friday 3pm" is Within(Clock, Weekday) and
+  # "christmas 2015" is Within(Holiday, Date 2015). Two atoms of the same size conflict.
+  #
+  # Tokens are read straight from the flat Array: @t[i] is a kind and @t[i + 1] a value, both nil past the
+  # end. A function word's Symbol is no other kind's value, so `@t[@i + 1] == :of` is the word "of".
+  # @api private
+  class Grammar
+    # h24: written with a leading zero, a 24-hour form (D4); seconds: written with seconds, or :hour for an hour alone
+    Clock = Struct.new(:hour, :min, :sec, :meridiem, :named, :h24, :seconds, :frac)
+    DayRef = Struct.new(:offset)
+    # Any of the three may be nil. With sep (:slash, :dash, :dot, :space), month and day are the first and
+    # second numbers as written and resolve picks the order (D10, D37); short: a two-digit year (D23)
+    Date = Struct.new(:year, :month, :day, :sep, :short)
+    Weekday = Struct.new(:wday)
+    Part = Struct.new(:name)
+    Season = Struct.new(:name)
+    Holiday = Struct.new(:name)
+    Weekend = Struct.new(:count)
+    Fiscal = Struct.new(:year)
+    IsoWeek = Struct.new(:week, :day) # D13: ISO week 1-53, and its day 1-7 (Monday 1) or nil
+    # A decade or century from its first year; short: a decade's two digits, for the 50-year window (D12)
+    Years = Struct.new(:unit, :year, :short)
+    # A unit, alone ("next week") or counted ("next 3 days", "3 days" in "3 days ago")
+    Unit = Struct.new(:unit, :count)
+    Rel = Struct.new(:word, :node)              # next / this / last / past / coming
+    Shift = Struct.new(:count, :unit, :anchor)  # signed count of units from the anchor, or from now
+    Counted = Struct.new(:n, :node, :anchor)    # the nth node after the anchor or now (n < 0: before)
+    Closest = Struct.new(:n, :node, :anchor)    # the nth node nearest the anchor, either side
+    Nth = Struct.new(:n, :node)                 # the nth (n = -1: the last) node inside its outer period
+    Within = Struct.new(:inner, :outer)
+    Range = Struct.new(:from, :to)
+    Open = Struct.new(:dir, :node)              # :after, :before, :since, :until, :upto, :from, :through
+    Lasting = Struct.new(:unit, :start)         # a counted Unit from the start's start: "3 days starting monday"
+    Holding = Struct.new(:unit, :node)          # the calendar unit that holds the node: "the week of the 18th"
+    Approx = Struct.new(:node)
+    Fuzzy = Struct.new(:pos, :node)             # :early, :mid, :late, :start, :end
+    Zoned = Struct.new(:node, :offset, :numeric)
+    Now = Struct.new
+    NOW = Now.new.freeze
+    WEEKEND = Weekend.new(nil).freeze
+    # Each node's class as a Symbol, to dispatch on in one step: a case over the classes tests them in turn
+    KIND = { Clock => :clock, DayRef => :day_ref, Date => :date, Weekday => :weekday, Part => :part, Season => :season,
+             Holiday => :holiday, Weekend => :weekend, Fiscal => :fiscal, Unit => :unit, Rel => :rel, Shift => :shift,
+             Nth => :nth, Within => :within, Range => :range, Open => :open, Approx => :approx, Fuzzy => :fuzzy,
+             Zoned => :zoned, Now => :now, Years => :years, IsoWeek => :iso_week, Counted => :counted,
+             Lasting => :lasting, Holding => :holding, Closest => :closest }.freeze
+
+    L = Lexer
+    S = Lexer::STRIDE
+    OPEN = { after: :after, before: :before, since: :since, until: :until, by: :until, upto: :upto, through: :until,
+             onafter: :from, onbefore: :through }.freeze
+    RANGE = { to: true, through: true, until: true, upto: true, butbefore: true }.freeze # D48: "X up to Y"
+    BEFORE_DAY = { the: true, from: true, between: true }.freeze # words that make "ninth" a day
+    SHIFT_TAIL = { ago: true, hence: true, later: true, from: true, after: true, before: true }.freeze
+    # The kinds item looks at before taking an atom; any other token starts one
+    ITEM_SPECIAL = Array.new(L::DECADE + 1) { [L::AT, L::WORD, L::ZONE, L::PLUS, L::DASH, L::DOT, L::SLASH].include?(_1) }.freeze
+    UNIT_SIZE = { second: 0, minute: 0, hour: 0, day: 2, business_day: 2, week: 3, fortnight: 3, month: 5,
+                  quarter: 6, season: 6, year: 7, decade: 8, century: 9 }.freeze
+
+    def self.parse(tokens) = new(tokens).phrase
+
+    FIRST = 16
+
+    # The longest prefix of the tokens that is a phrase (or with duration: true, a duration), as [tree, its
+    # length in Array entries], or nil. A failure says which token the parser stuck at; a longer prefix gets
+    # no further, so the next try ends before it. Each try is a new Array: a lookahead past the end must
+    # read nil. A long run of words is tried on its first FIRST tokens before all of them: most fail well
+    # inside that, as a longer prefix would, and the parse of a long run costs its length. spent: an Array
+    # whose first entry counts the entries parsed.
+    def self.longest(tokens, duration: false, spent: nil)
+      if tokens.size > FIRST * S
+        spent[0] += FIRST * S if spent
+        g = new(tokens[0, FIRST * S])
+        tree = duration ? g.duration : g.phrase
+        tokens = tokens[0, g.stuck] if Failure === tree && g.stuck < (FIRST - 4) * S
+      end
+      until tokens.empty?
+        spent[0] += tokens.size if spent
+        g = new(tokens)
+        tree = duration ? g.duration : g.phrase
+        return [tree, tokens.size] unless Failure === tree
+        stop = g.stuck < tokens.size ? g.stuck : tokens.size - S
+        tokens = tokens[0, stop]
+      end
+    end
+
+    def initialize(tokens)
+      @t = tokens
+      @n = tokens.size
+      @i = 0
+      @approx = false
+    end
+
+    attr_reader :stuck
+
+    def phrase
+      return failure(:incomplete) if @n == 0
+      node = top
+      return node if Failure === node
+      return failure(:conflict) if @i < @n
+      return failure(:incomplete) if Unit === node
+      @approx ? Approx.new(node) : node
+    end
+
+    private
+
+    def top
+      w = @t[@i + 1]
+      if w == :anytime && @t[@i] == L::WORD && OPEN[@t[@i + S + 1]] # D48: "anytime after 2014"
+        @i += S
+        w = @t[@i + 1]
+      end
+      if ((dir = OPEN[w]) || starting?(@i)) && @i + S < @n
+        dir ||= :from
+        @i += S
+        @i += S if dir == :from && @t[@i + 1] == :from # "starting from 1/1/2016"
+        # D48: "after 2 days, 3 hours and 1 minute" is a shift, as "in 2 days, 3 hours and 1 minute"
+        if dir == :after && @t[@i] == L::NUM && unit_at(@i + S) && @t[@i + 2 * S] == L::NUM && unit_at(@i + 3 * S)
+          q = quantity
+          return shifts(1, q, more_units(q), nil)
+        end
+        node = body
+        return node if Failure === node
+        return between(dir, node) if @but
+        # "after 3 hours" is a shift, not an open end
+        return Shift.new(node.count, node.unit, nil) if Unit === node && node.count && dir == :after
+        return failure(:incomplete) if Unit === node
+        Open.new(dir, node)
+      elsif @t[@i] == L::WORD && w == :around
+        @i += S
+        @approx = true
+        body
+      else
+        (w == :for && for_from) || open_after(body(w))
+      end
+    end
+
+    def body(w = @t[@i + 1])
+      open = @t[@i] == L::WORD && (w == :between || w == :from) # the kind first: Integer == Symbol is slow
+      if open
+        start = @i
+        @i += S
+      end
+      fs = @i
+      @mid = nil
+      from = item(open)
+      return from if Failure === from
+      if open
+        return lasting(from) if w == :from && @t[@i + 1] == :for && @t[@i] == L::WORD
+        return @i = start && failure(:incomplete) unless separator?(@t[@i + 1] == :and)
+      elsif @mid
+        return failure(:incomplete) unless separator?(@mid == :between)
+      else
+        # D48: "today and tomorrow": two days in a row are both of them
+        return from unless separator?(DayRef === from && @t[@i + S] == L::DAY && @t[@i + S + 1] == from.offset + 1)
+      end
+      fe = @i
+      @but = @t[@i + 1] == :butbefore
+      @i += S
+      ts = @i
+      to = day_and_year(from, ts)
+      return to if to
+      to = item(true)
+      return to if Failure === to
+      range(from, to, fs, fe, ts)
+    end
+
+    def separator?(and_ok)
+      return false if @i >= @n
+      k = @t[@i]
+      k == L::DASH || (k == L::WORD && (RANGE[@t[@i + 1]] || (and_ok && @t[@i + 1] == :and))) || k == L::SLASH
+    end
+
+    # One constituent after another, then nested by size. Most items are one or two atoms, so the lists of
+    # atoms and of where each starts are made only for a third.
+    def item(loose)
+      last = nil
+      a1 = s1 = a2 = s2 = nil
+      atoms = nil
+      starts = nil
+      dates = 0
+      zone = nil
+      numeric = false
+      at = false
+      signed = false
+      while @i < @n
+        k = @t[@i]
+        v = @t[@i + 1]
+        a = nil
+        if !ITEM_SPECIAL[k]
+          # a token that only starts an atom, the commonest case: none of the tests below apply
+        elsif k == L::AT
+          @i += S
+          at = true
+          next
+        elsif k == L::WORD && (v == :the || v == :on || (v == :ad && Date === last)) # D44: "1973 A.D."
+          @i += S
+          next
+        elsif k == L::WORD && v == :of && Clock === last && day_at?(@i + S)
+          @i += S # D48: "from 5 to 6pm of April 22", as "on April 22"
+          next
+        elsif k == L::WORD && (v == :for || v == :during)
+          break if v == :for && !for_when?(@i + S) # D48: "from 4pm for 30 minutes": a length; lasting reads it
+          @i += S # D48: "for 9am", "for the past month", "during March": the time it is for
+          next
+        elsif k == L::WORD && v == :around
+          @approx = true # D14: "tomorrow around 10am", "3pm approximately", "11ish"
+          @i += S
+          next
+        elsif k == L::WORD && (v == :from || v == :between) && last && !@mid
+          # D44: "tomorrow from 9 to 5", "friday between 1 and 4pm": the range is within what came before
+          @mid = v
+          @i += S
+          next
+        elsif k == L::ZONE
+          @i += S
+          next if zone # D44: "GMT+0800 (CST)", "-0500 (CDT)": a second zone labels the first
+          zone = v
+          # "GMT-1", "UTC +8"; with spaces both sides it is a range ("15:00 GMT - 18:00 GMT")
+          if Integer === zone && @i + S < @n && (@t[@i] == L::PLUS || @t[@i] == L::DASH) &&
+             (@t[@i + 3] == @t[@i - 1] || @t[@i + S + 3] == @t[@i + 4]) && (off = offset(@i + S))
+            zone += @t[@i] == L::PLUS ? off : -off
+            return failure(:invalid_date, @i + S) if zone.abs >= 86_400 # D18: "JST+23:00"
+            numeric = true
+            @i += 2 * S
+          end
+          next
+        elsif (k == L::PLUS || k == L::DASH) && @i + S < @n && signed?(last)
+          signed = true
+          break
+        elsif (k == L::PLUS || (k == L::DASH && ((machine?(last) && @t[@i + 4] == @t[@i + S + 3]) || dash_offset?(last)))) && @i + S < @n &&
+              (off = offset(@i + S))
+          zone = k == L::PLUS ? off : -off
+          numeric = true
+          @i += 2 * S
+          next
+        elsif (k == L::DASH || k == L::DOT) && Date === last && last.sep && last.day && @t[@i + S] == L::CLOCK &&
+              @t[@i + 3] == @t[@i - 1] && @t[@i + 4] == @t[@i + S + 3]
+          @i += S # D44: "05/31/2024-14:15": a clock joined to a numeric date by a dash is its time, as after a T; D48: or a dot
+          next
+        elsif k == L::DASH && Date === last && last.sep && last.year && glued_clock?(@i)
+          # D48: "05/31/2024:14:15": the colon after a year lexes as a dash, so the clock comes as two numbers
+          start = @i + S
+          a = Clock.new(@t[@i + S + 1], @t[@i + 3 * S + 1], 0, nil, nil, true, false)
+          @i += 4 * S
+        elsif k == L::SLASH && machine?(last) && @t[@i + S] == L::NUM && @t[@i + S + 2] & L::DIGITS == 4
+          break # D48: "2015-03-28 17:00:00/2015-03-29 21:00:00", an ISO 8601 interval
+        elsif k == L::DASH || (k == L::WORD && (RANGE[v] || v == :and || v == :or || v == :orlater || v == :orearlier))
+          break
+        elsif k == L::WORD && (v == :before || v == :after) && last && clock_at?(@i + S)
+          break # D48: "tomorrow after 4pm", clipped by open_after
+        end
+        unless a
+          start = @i
+          a = atom(loose || at || !last.nil?)
+          return a if Failure === a
+        end
+        if atoms
+          atoms << a
+          starts << start
+        elsif a2
+          atoms = [a1, a2, a]
+          starts = [s1, s2, start]
+        elsif a1
+          a2 = a
+          s2 = start
+        else
+          a1 = a
+          s1 = start
+        end
+        last = a
+        dates += 1 if Date === a
+        at = false
+      end
+      return signed_shifts(nil) if signed && last.nil?
+      return failure(:incomplete) if at || last.nil?
+      node = last
+      if a2 && !atoms && dates < 2
+        node = pair(a1, s1, a2, s2)
+      elsif a2
+        atoms ||= [a1, a2]
+        starts ||= [s1, s2]
+        merge_year(atoms, starts) if dates > 1
+        node = combine(atoms, starts)
+      end
+      return node if Failure === node
+      node = Zoned.new(node, zone, numeric) if zone
+      signed ? signed_shifts(node) : node
+    end
+
+    # "for" before a time or a period, not a length (E2)
+    def for_when?(i)
+      k = @t[i]
+      clock_at?(i) || (k == L::WORD && @t[i + 1] == :the) || k == L::REL ||
+        (k == L::ART && @t[i + S] == L::UNIT && clock_word?(i + 2 * S))
+    end
+
+    def day_at?(i)
+      k = @t[i]
+      k == L::MONTH || k == L::DAY || (k == L::NUM && @t[i + S] == L::SLASH)
+    end
+
+    def clock_at?(i)
+      k = @t[i]
+      k == L::CLOCK || k == L::NAMED || (k == L::NUM && @t[i + S] == L::MER)
+    end
+
+    def glued_clock?(i)
+      t = @t
+      t[i + S] == L::NUM && t[i + 2 * S] == L::DASH && t[i + 3 * S] == L::NUM && t[i - 1] == t[i + 3] && t[i + 4] == t[i + S + 3] &&
+        t[i + S + 4] == t[i + 2 * S + 3] && t[i + 2 * S + 4] == t[i + 3 * S + 3] && t[i + S + 2] & L::DIGITS == 2 &&
+        t[i + 3 * S + 2] & L::DIGITS == 2 && t[i + S + 1] <= 23 && t[i + 3 * S + 1] <= 59
+    end
+
+    def machine?(a)
+      a = a.inner if Within === a
+      Clock === a && (a.seconds || a.h24)
+    end
+
+    # D42: "15:45 -0800", "11:00 -03:00": after a clock, a dash with a space before it and none after, then
+    # four digits or hh:mm, is an offset; "3:00 - 5:00" and "9-5" stay ranges
+    def dash_offset?(last)
+      last = last.inner if Within === last # "2010-09-12 3:12pm -0700"
+      return false unless Clock === last && @i + S < @n && @t[@i + 3] > @t[@i - 1] && @t[@i + S + 3] == @t[@i + 4]
+      k = @t[@i + S]
+      (k == L::NUM && @t[@i + S + 2] & L::DIGITS == 4) || (k == L::CLOCK && @t[@i + S + 2] & L::DIGITS == 2)
+    end
+
+    # "+01:00", "-0400", "+1"; D18: up to 23:59
+    def offset(i)
+      k = @t[i]
+      v = @t[i + 1]
+      if k == L::CLOCK && v % 100 == 0 then hm(v / 10_000, v / 100 % 100)
+      elsif k == L::NUM && @t[i + 2] & L::DIGITS == 4 then hm(v / 100, v % 100)
+      elsif k == L::NUM && @t[i + 2] & L::DIGITS <= 2 && v <= 14 then v * 3600
+      end
+    end
+
+    def hm(h, m) = h <= 23 && m <= 59 ? h * 3600 + m * 60 : nil
+
+    # Points at token index i, or at the end of the text when the tokens ran out
+    def failure(code, i = @i)
+      @stuck = i > @i ? i : @i
+      if i < @n
+        Failure.new(code, @t[i + 3], @t[i + 4])
+      else
+        stop = @n == 0 ? 0 : @t[@n - 1]
+        Failure.new(code, stop, stop)
+      end
+    end
+  end
+end
+
+require_relative "grammar/nesting"
+require_relative "grammar/ranges"
+require_relative "grammar/words"
+require_relative "grammar/ordinals"
+require_relative "grammar/clock_words"
+require_relative "grammar/numbers"
+require_relative "grammar/compact"
+require_relative "grammar/shifts"
+require_relative "grammar/duration"
+require_relative "grammar/rule"