plume 0.2.0 → 0.3.0

data/lib/plume/parser.rb

@@ -10,16 +10,29 @@ module Plume
 				@sql, @token, @beg, @fin, @expected = sql, token, beg, fin, expected
 			end
 
+			def to_error
+				# At the end of input the span sits one byte past the SQL
+				location = [@beg, [@fin, @sql.bytesize].min].freeze
+				SyntaxError.new(msg, location:, token: @token, line:, expected: @expected.freeze)
+			end
+
+			def line = @sql.b.byteslice(0, @beg).count("\n") + 1
+
 			def msg
-				line = @sql.byteslice(0, @beg).count("\n") + 1
-				where = @sql.strip.include?("\n") ? " line #{line}" : ""
+				src = @sql.b # the SQL can be invalid UTF-8, which String#strip will not take
+				where = src.strip.include?("\n") ? " line #{line}" : ""
 				what = @token ? "token #{@token}[#{@sql.byteslice(@beg, @fin - @beg).inspect}]" : "end of input"
 				"Unexpected #{what} at#{where}:\n#{Plume.excerpt(@sql, @beg, @fin)}\n  Expected one of: #{@expected.join(", ")}\n"
 			end
 		end
 
 		# see: https://github.com/sqlite/sqlite/blob/master/src/parse.y#L255-L283
+		# Literals in an expression, names anywhere else: SQLite has no TRUE or FALSE keyword, and lets
+		# the CURRENT_* ones (CTIME_KW) fall back to names
+		LITERAL_KEYWORDS = Set[:TRUE, :FALSE, :CURRENT_DATE, :CURRENT_TIME, :CURRENT_TIMESTAMP].freeze
+
 		TOKEN_FALLBACKS = Set[
+			*LITERAL_KEYWORDS,
 			:EXPLAIN,
 			:QUERY,
 			:PLAN,
@@ -57,6 +70,7 @@ module Plume
 			:GLOB,
 			:LIKE,
 			:ROWID,
+			:STORED,
 			:STRICT,
 			:COLUMN,
 			:DO,
@@ -147,11 +161,11 @@ module Plume
 			result = catch(:ERROR) { sql_stmt_list }
 
 			if ErrorMessage === result
-				raise SyntaxError.new(result.msg)
+				raise result.to_error
 			else
 				result
 			end
-		rescue SystemStackError
+		rescue *STACK_EXHAUSTED
 			# How deep is too deep depends on the stack the caller has left, so no fixture relies on it
 			raise SyntaxError.new("statement nested too deeply")
 		end
@@ -651,6 +665,11 @@ module Plume
 				#   cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist
 				#   cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm
 				#   cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm
+				#   cmd ::= ALTER TABLE fullname DROP CONSTRAINT nm
+				#   cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm DROP NOT NULL
+				#   cmd ::= ALTER TABLE fullname ALTER kwcolumn_opt nm SET NOT NULL onconf
+				#   cmd ::= ALTER TABLE fullname ADD CONSTRAINT nm CHECK LP expr RP onconf
+				#   cmd ::= ALTER TABLE fullname ADD CHECK LP expr RP onconf
 				#   kwcolumn_opt ::= . | COLUMNKW
 				#
 				# Relevant SQLite documentation:
@@ -669,26 +688,69 @@ module Plume
 				column_tk = name
 				to_kw = require :TO
 				new_name_tk = name
+			elsif current_token == :ADD && (peek in :CONSTRAINT | :CHECK)
+				action = :ADD_CONSTRAINT
+				action_kw = require :ADD
+				constraint_name = name if (constraint_kw = maybe :CONSTRAINT)
+				constraint = check_table_constraint(constraint_kw, constraint_name)
 			elsif (action_kw = maybe_one_of [:ADD, :COLUMN].freeze, :ADD)
 				action = :ADD_COLUMN
 				column_definition = column_def
+			elsif current_token == :DROP && peek == :CONSTRAINT
+				action = :DROP_CONSTRAINT
+				action_kw = require_all_of :DROP, :CONSTRAINT
+				constraint_name_tk = name
 			elsif (action_kw = maybe_one_of [:DROP, :COLUMN].freeze, :DROP)
 				action = :DROP_COLUMN
 				column_tk = name
+			elsif (action_kw = maybe_one_of [:ALTER, :COLUMN].freeze, :ALTER)
+				column_tk = name
+				if current_token == :DROP
+					action = :DROP_NOT_NULL
+					not_null_kw = require_all_of :DROP, :NOT, :NULL
+				else
+					action = :SET_NOT_NULL
+					not_null_kw = require_all_of :SET, :NOT, :NULL
+					on_conflict = maybe { conflict_clause }
+				end
 			else
-				expected!(:RENAME, :ADD, :DROP)
+				expected!(:RENAME, :ADD, :DROP, :ALTER)
 			end
 
 			AlterTableStatement.concrete(
-				full_source:       @lexer.sql,
-				alter_table_kw:    Token::Keyword(alter_table_kw),
-				table:             table,
-				action:            action,
-				action_kw:         Token::Keyword(action_kw),
-				column_tk:         Token::Identifier(column_tk),
-				to_kw:             Token::Keyword(to_kw),
-				new_name_tk:       Token::Identifier(new_name_tk),
-				column_definition: column_definition,
+				full_source:        @lexer.sql,
+				alter_table_kw:     Token::Keyword(alter_table_kw),
+				table:              table,
+				action:             action,
+				action_kw:          Token::Keyword(action_kw),
+				column_tk:          Token::Identifier(column_tk),
+				to_kw:              Token::Keyword(to_kw),
+				new_name_tk:        Token::Identifier(new_name_tk),
+				column_definition:  column_definition,
+				constraint:         constraint,
+				constraint_name_tk: Token::Identifier(constraint_name_tk),
+				not_null_kw:        Token::Keyword(not_null_kw),
+				conflict_clause:    on_conflict,
+			)
+		end
+
+		# tcons ::= CHECK LP expr RP onconf, after the `CONSTRAINT nm` the caller may have read
+		def check_table_constraint(constraint_kw = nil, constraint_name = nil)
+			check_kw = require :CHECK
+			check_lp = require :LP
+			check = expression
+			check_rp = require :RP
+			on_conflict = maybe { conflict_clause }
+
+			CheckTableConstraint.concrete(
+				full_source:     @lexer.sql,
+				constraint_kw:   Token::Keyword(constraint_kw),
+				name_tk:         Token::Identifier(constraint_name),
+				check_kw:        Token::Keyword(check_kw),
+				check_lp:        Token::Punctuation(check_lp),
+				expression:      check,
+				check_rp:        Token::Punctuation(check_rp),
+				conflict_clause: on_conflict,
 			)
 		end
 
@@ -874,8 +936,11 @@ module Plume
 					select_statement: as_select
 				)
 			elsif (columns_lp = maybe :LP)
-				columns         = require_some(trailing_sep: :COMMA) { column_def }
-				# inlined `table_constraint`
+				# The comma after the last column leads into the table constraints: `columnlist COMMA conslist`
+				columns         = require_some(given: column_def, trailing_sep: :COMMA) do
+					column_def unless current_token in :CONSTRAINT | :PRIMARY | :UNIQUE | :CHECK | :FOREIGN
+				end
+				# inlined `table_constraint`, but for CHECK, which ALTER TABLE shares through check_table_constraint
 				constraints     = maybe_some(trailing_sep: [:COMMA, nil].freeze) do
 					#
 						# Relevant `parse.y` grammar rules:
@@ -895,7 +960,7 @@ module Plume
 						#      │                               └───────{ , }◀──────┘└────────▶─────────┘                              │
 						#      ├─▶{ UNIQUE }──▶{ ( }┬▶[ indexed-column ]┬▶{ ) }─▶[ conflict-clause ]────────────────────────────────▶─┤
 						#      │                    └───────{ , }◀──────┘                                                             │
-						#      ├─▶{ CHECK }─▶{ ( }─▶[ expr ]─▶{ ) }─────────────────────────────────────────────────────────────────▶─┤
+						#      ├─▶{ CHECK }─▶{ ( }─▶[ expr ]─▶{ ) }─▶[ conflict-clause ]────────────────────────────────────────────▶─┤
 						#      └─▶{ FOREIGN }─▶{ KEY }─▶{ ( }┬▶[ indexed-column ]┬▶{ ) }─▶[ foreign-key-clause ]────────────────────▶─┘
 						#                                    └──────{ , }◀───────┘
 						#
@@ -911,9 +976,9 @@ module Plume
 						constraint_name     = name
 					end
 
-					if    (unique_kw      = maybe          :UNIQUE)
+					if    (unique_kw      = maybe :UNIQUE)
 						unique_lp           = require :LP
-						unique_columns      = require_some(trailing_sep: :COMMA) { indexed_column }
+						unique_columns      = require_some(trailing_sep: :COMMA) { indexed_column(nulls: true) }
 						unique_rp           = require        :RP
 						on_conflict         = maybe        { conflict_clause }
 
@@ -927,23 +992,11 @@ module Plume
 							unique_rp:       Token::Punctuation(unique_rp),
 							conflict_clause: on_conflict,
 						)
-					elsif (check_kw       = maybe          :CHECK)
-						check_lp            = require        :LP
-						check               =                expression
-						check_rp            = require        :RP
-
-						CheckTableConstraint.concrete(
-							full_source:     @lexer.sql,
-							constraint_kw:   Token::Keyword(constraint_kw),
-							name_tk:         Token::Identifier(constraint_name),
-							check_kw:        Token::Keyword(check_kw),
-							check_lp:        Token::Punctuation(check_lp),
-							expression:      check,
-							check_rp:        Token::Punctuation(check_rp),
-						)
+					elsif current_token == :CHECK
+						check_table_constraint(constraint_kw, constraint_name)
 					elsif (primary_key_kw = maybe_all_of   :PRIMARY, :KEY)
 						primary_key_lp      = require :LP
-						primary_key_columns = require_some(trailing_sep: :COMMA) { indexed_column }
+						primary_key_columns = require_some(trailing_sep: :COMMA) { indexed_column(nulls: true) }
 						autoincrement_kw    = maybe          :AUTOINCREMENT
 						primary_key_rp      = require        :RP
 						on_conflict         = maybe        { conflict_clause }
@@ -1457,7 +1510,8 @@ module Plume
 			left = basic_expression
 
 			while (operator = current_token)
-				operator_precedence = OPERATOR_PRECEDENCE[operator] || -1
+				# ~ has a precedence for its operand's sake, but is no binary operator
+				operator_precedence = ((operator != :BITNOT) && OPERATOR_PRECEDENCE[operator]) || -1
 				break if operator_precedence < min_precedence
 				next_precedence = operator_precedence + 1
 
@@ -1497,39 +1551,20 @@ module Plume
 						else
 							expected!(:RP, "select-stmt", "expr")
 						end
-					elsif (ref = maybe { identifier })
-						if maybe :DOT
-							name = identifier
+					elsif (first_tk = maybe { name })
+						# nm dbnm paren_exprlist: a table, or a table-valued function's call
+						dot_tk = maybe :DOT
+						schema_tk, object_tk = dot_tk ? [first_tk, name] : [nil, first_tk]
 
-							if (arguments_lp = maybe :LP)
-								arguments = maybe_some(trailing_sep: :COMMA) { expression }
-								arguments_rp = require :RP
-
-								right = FunctionReference.concrete(
-									full_source: @lexer.sql,
-									schema_tk: Token::Identifier(ref),
-									function_tk: Token::Identifier(name),
-									arguments_lp: Token::Punctuation(arguments_lp),
-									arguments: FunctionArguments.concrete(
-										full_source: @lexer.sql,
-										expressions: arguments
-									),
-									arguments_rp: Token::Punctuation(arguments_rp),
-								)
-							else
-								right = TableName.concrete(
-									full_source: @lexer.sql,
-									schema_tk: Token::Identifier(ref),
-									table_tk: Token::Identifier(name),
-								)
-							end
-						elsif (arguments_lp = maybe :LP)
+						if (arguments_lp = maybe :LP)
 							arguments = maybe_some(trailing_sep: :COMMA) { expression }
 							arguments_rp = require :RP
 
 							right = FunctionReference.concrete(
 								full_source: @lexer.sql,
-								function_tk: Token::Identifier(ref),
+								schema_tk: Token::Identifier(schema_tk),
+								dot_tk: Token::Punctuation(dot_tk),
+								function_tk: Token::Identifier(object_tk),
 								arguments_lp: Token::Punctuation(arguments_lp),
 								arguments: FunctionArguments.concrete(
 									full_source: @lexer.sql,
@@ -1538,15 +1573,15 @@ module Plume
 								arguments_rp: Token::Punctuation(arguments_rp),
 							)
 						else
-							table = ref
-
 							right = TableName.concrete(
 								full_source: @lexer.sql,
-								table_tk: Token::Identifier(table),
+								schema_tk: Token::Identifier(schema_tk),
+								dot_tk: Token::Punctuation(dot_tk),
+								table_tk: Token::Identifier(object_tk),
 							)
 						end
 					else
-						expected!(:LP, :ID)
+						expected!(:LP, :STRING, "idj")
 					end
 
 					left = BinaryExpression.concrete(
@@ -1801,6 +1836,13 @@ module Plume
 						not_null_kw:     Token::Keyword(not_null_kw),
 						conflict_clause: on_conflict,
 					)
+				elsif current_token == :DEFERRABLE || (current_token == :NOT && peek == :DEFERRABLE)
+					DeferrableColumnConstraint.concrete(
+						full_source:     @lexer.sql,
+						constraint_kw:   Token::Keyword(constraint_kw),
+						name_tk:         Token::Identifier(constraint_name),
+						deferrable_span: Token::Keyword(defer_subclause),
+					)
 				elsif (null_kw        = maybe        :NULL)
 					on_conflict         = maybe      { conflict_clause }
 
@@ -2083,7 +2125,7 @@ module Plume
 
 		# The inside of a window-defn parenthesis, shared by OVER (…) and WINDOW name AS (…)
 		def window_body
-			base_window_name_tk = maybe { identifier(except: [:PARTITION, :ORDER, :RANGE, :ROWS, :GROUPS]) }
+			base_window_name_tk = maybe { name(except: [:PARTITION, :ORDER, :RANGE, :ROWS, :GROUPS]) }
 			if (partition_by_kw = maybe_all_of(:PARTITION, :BY))
 				partitions = require_some(trailing_sep: :COMMA) { expression }
 			end
@@ -2184,7 +2226,7 @@ module Plume
 					window_rp: Token::Punctuation(window_rp),
 				)
 			else
-				window_name_tk = identifier
+				window_name_tk = name
 
 				OverClause.concrete(
 					full_source: @lexer.sql,
@@ -2245,9 +2287,12 @@ module Plume
 				columns = require_some(trailing_sep: :COMMA) { indexed_column }
 				columns_rp = require :RP
 			end
-			foreign_key_actions = maybe_some(trailing_sep: nil) do
+			# refargs ::= refargs refarg, so ON and MATCH clauses come in any order
+			refargs = maybe_some(trailing_sep: nil) do
+				next match_clause if current_token == :MATCH
+
 				on_kw      = require        :ON
-				trigger_kw = require_one_of :DELETE, :UPDATE
+				trigger_kw = require_one_of :DELETE, :UPDATE, :INSERT
 				action_kw  = require_one_of :CASCADE, :RESTRICT, [:SET, :NULL].freeze, [:SET, :DEFAULT].freeze, [:NO, :ACTION].freeze
 				ForeignKeyAction.concrete(
 					full_source: @lexer.sql,
@@ -2256,24 +2301,9 @@ module Plume
 					action_kw:   Token::Keyword(action_kw),
 				)
 			end
-			match_clauses = maybe_some(trailing_sep: nil) do
-				match_kw = require :MATCH
-				type_tk  = name(except: [:ON, :MATCH, :DEFERRABLE, :NOT])
-				MatchClause.concrete(
-					full_source: @lexer.sql,
-					match_kw:    Token::Keyword(match_kw),
-					type_tk:     Token::Identifier(type_tk),
-				)
-			end
-
-			deferrable_span = maybe_one_of(
-				[:NOT, :DEFERRABLE, :INITIALLY, :IMMEDIATE].freeze,
-				[:NOT, :DEFERRABLE, :INITIALLY, :DEFERRED].freeze,
-				[:DEFERRABLE, :INITIALLY, :IMMEDIATE].freeze,
-				[:DEFERRABLE, :INITIALLY, :DEFERRED].freeze,
-				[:NOT, :DEFERRABLE].freeze,
-				:DEFERRABLE,
-			)
+			foreign_key_actions = refargs.grep(ForeignKeyAction)
+			match_clauses = refargs.grep(MatchClause)
+			deferrable_span = defer_subclause
 
 			ForeignKeyClause.concrete(
 				full_source: @lexer.sql,
@@ -2288,6 +2318,28 @@ module Plume
 			)
 		end
 
+		def match_clause
+			match_kw = require :MATCH
+			type_tk  = name(except: [:ON, :MATCH, :DEFERRABLE, :NOT])
+			MatchClause.concrete(
+				full_source: @lexer.sql,
+				match_kw:    Token::Keyword(match_kw),
+				type_tk:     Token::Identifier(type_tk),
+			)
+		end
+
+		# defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt | DEFERRABLE init_deferred_pred_opt
+		def defer_subclause
+			maybe_one_of(
+				[:NOT, :DEFERRABLE, :INITIALLY, :IMMEDIATE].freeze,
+				[:NOT, :DEFERRABLE, :INITIALLY, :DEFERRED].freeze,
+				[:DEFERRABLE, :INITIALLY, :IMMEDIATE].freeze,
+				[:DEFERRABLE, :INITIALLY, :DEFERRED].freeze,
+				[:NOT, :DEFERRABLE].freeze,
+				:DEFERRABLE,
+			)
+		end
+
 		def conflict_clause
 			#
 				# Used by:
@@ -2403,7 +2455,7 @@ module Plume
 				expression_lp = require :LP
 				e = expression
 				as_kw = require :AS
-				t = type_name
+				t = type_name unless current_token == :RP # typetoken ::= .
 				expression_rp = require :RP
 
 				CastExpression.concrete(
@@ -2491,7 +2543,10 @@ module Plume
 					select_rp: Token::Punctuation(select_rp),
 				)
 			# A string followed by a dot is a quoted table name, as in `'t'.col`
-			elsif !(current_token == :STRING && peek == :DOT) && (literal_tk = maybe { literal_value })
+			# A string or TRUE/FALSE before a dot names a table: `'t'.x`, `false.false`. SQLite has no TRUE
+			# or FALSE keyword at all, while CURRENT_* is always a literal in an expression
+			elsif !((current_token in :STRING | :TRUE | :FALSE) && peek == :DOT) &&
+					(literal_tk = maybe { literal_value })
 				LiteralExpression.concrete(
 					full_source: @lexer.sql,
 					value_tk: literal_tk,
@@ -2561,7 +2616,12 @@ module Plume
 				raise_function
 			# SQLite lexes "#9" as a variable too, but "#" and a digit only ever names a register
 			# inside its own nested parses, so as a parameter it is a syntax error
-			elsif :VARIABLE == current_token && !current_value.match?(/\A#\d/)
+			elsif :VARIABLE == current_token && !current_value.b.match?(/\A#\d/)
+				# SQLite numbers them from ?1 up to its default SQLITE_MAX_VARIABLE_NUMBER
+				if (number = current_value.b[/\A\?(\d+)\z/, 1]) && !number.to_i.between?(1, 32_766)
+					expected!("?1 to ?32766")
+				end
+
 				Variable.concrete(
 					full_source: @lexer.sql,
 					name_tk: Token::Variable(require(:VARIABLE)),
@@ -2681,38 +2741,22 @@ module Plume
 				#
 				# [description]
 			#
-			if (distinct_kw = maybe_one_of :DISTINCT, :ALL)
-				expressions = require_some(trailing_sep: :COMMA) { expression }
-				if (order_by_kw = maybe_all_of :ORDER, :BY)
-					order_by = require_some(trailing_sep: :COMMA) { ordering_term }
-				end
-
-				FunctionArguments.concrete(
-					full_source: @lexer.sql,
-					distinct_kw: Token::Keyword(distinct_kw),
-					expressions: expressions,
-					order_by_kw: Token::Keyword(order_by_kw),
-					order_by: order_by,
-				)
-			elsif maybe :STAR
-				StarFunctionArgument.new
-			elsif (e = maybe { expression })
-				expressions = require_some(given: e, trailing_sep: :COMMA) { expression }
-				if (order_by_kw = maybe_all_of :ORDER, :BY)
-					order_by = require_some(trailing_sep: :COMMA) { ordering_term }
-				end
+			return StarFunctionArgument.new if maybe :STAR
 
-				FunctionArguments.concrete(
-					full_source: @lexer.sql,
-					expressions: expressions,
-					order_by_kw: Token::Keyword(order_by_kw),
-					order_by: order_by,
-				)
-			elsif :RP == current_token
-				EmptyFunctionArgument.new
-			else
-				expected!(:DISTINCT, :STAR, :RP, "expr")
+			distinct_kw = maybe_one_of :DISTINCT, :ALL
+			expressions = maybe_some(trailing_sep: :COMMA) { expression }
+			if (order_by_kw = maybe_all_of :ORDER, :BY)
+				order_by = require_some(trailing_sep: :COMMA) { ordering_term }
 			end
+			return EmptyFunctionArgument.new unless distinct_kw || expressions.any? || order_by_kw
+
+			FunctionArguments.concrete(
+				full_source: @lexer.sql,
+				distinct_kw: Token::Keyword(distinct_kw),
+				expressions: expressions,
+				order_by_kw: Token::Keyword(order_by_kw),
+				order_by: order_by,
+			)
 		end
 
 		def frame_spec
@@ -2745,7 +2789,7 @@ module Plume
 				if (between_kw = maybe :BETWEEN)
 					if (unbounded_kw = maybe :UNBOUNDED)
 						precedence = 1
-						preceding_kw = maybe :PRECEDING
+						preceding_kw = require :PRECEDING
 						starting_boundary = FrameBoundary.concrete(
 							full_source: @lexer.sql,
 							type_tk: Token::Keyword(preceding_kw),
@@ -2786,7 +2830,7 @@ module Plume
 							type_tk: Token::Keyword(current_row_kw),
 						)
 					elsif (unbounded_kw = maybe :UNBOUNDED)
-						following_kw = maybe :FOLLOWING
+						following_kw = require :FOLLOWING
 						ending_boundary = FrameBoundary.concrete(
 							full_source: @lexer.sql,
 							type_tk: Token::Keyword(following_kw),
@@ -2814,7 +2858,7 @@ module Plume
 						expected!("CURRENT ROW", "UNBOUNDED FOLLOWING", "expr")
 					end
 				elsif (unbounded_kw = maybe :UNBOUNDED)
-					preceding_kw = maybe :PRECEDING
+					preceding_kw = require :PRECEDING
 					starting_boundary = FrameBoundary.concrete(
 						full_source: @lexer.sql,
 						type_tk: Token::Keyword(preceding_kw),
@@ -3474,7 +3518,9 @@ module Plume
 			end
 		end
 
-		def indexed_column
+		# `nulls`: whether NULLS FIRST/LAST may follow, as in the sortlist of a PRIMARY KEY or UNIQUE -
+		# not in the eidlist of a foreign key, a view or a CTE
+		def indexed_column(nulls: false)
 			# Used by:
 			#   - ccons, tcons, creat-view, cte
 			#
@@ -3483,6 +3529,7 @@ module Plume
 			#   collate ::= . | COLLATE ids.
 			#   sortorder ::= . | ASC | DESC
 			#   nm ::= idj | STRING
+			#   (PRIMARY KEY and UNIQUE take a sortlist, whose items may also end NULLS FIRST or NULLS LAST)
 			#
 			# Syntax diagram:
 			#   ◯──▶{ column-name }─┬─▶───────────────────────────────┬┬────────▶───┬──▶◯
@@ -3490,7 +3537,7 @@ module Plume
 			#                                                          └─▶{ DESC }──┘
 			#
 			# Simplified grammar:
-			#   name [COLLATE identifier] [ASC | DESC]
+			#   name [COLLATE identifier] [ASC | DESC] [NULLS FIRST | NULLS LAST]
 			#
 			# Relevant SQLite documentation:
 			#   - https://www.sqlite.org/syntax/indexed-column.html
@@ -3512,49 +3559,15 @@ module Plume
 				collation_tk = identifier_or_string(except: [:ASC, :DESC].freeze)
 			end
 			direction_tk = maybe_one_of :ASC, :DESC
+			nulls_tk     = maybe_one_of [:NULLS, :FIRST].freeze, [:NULLS, :LAST].freeze if nulls
 
 			IndexedColumn.concrete(
 				full_source: @lexer.sql,
 				name_tk:      Token::Identifier(column_name),
 				collate_kw:   Token::Keyword(collate_kw),
 				collation_tk: Token::Keyword(collation_tk),
-				direction_tk: Token::Keyword(direction_tk)
-			)
-		end
-
-		def indexed_expression
-			# Used by:
-			#   - tcons, upsert, expr (ORDER BY), create-index, window
-			#
-			# Relevant `parse.y` grammar rules:
-			#   sortlist  ::= sortlist COMMA expr sortorder nulls | expr sortorder nulls
-			#   sortorder ::= . | ASC | DESC
-			#   nulls     ::= . | NULLS FIRST | NULLS LAST
-			#
-			# Syntax diagram:
-			#   ◯─▶[ expr ]─┬────────▶───┬─┬────────────▶────────────┬───▶◯
-			#               ├─▶{ ASC }─▶─┤ ├─▶{ NULLS }─▶{ FIRST }─▶─┤
-			#               └─▶{ DESC }──┘ └─▶{ NULLS }─▶{ LAST }────┘
-			#
-			# Simplified grammar:
-			#   expression [ASC | DESC] [NULLS FIRST | NULLS LAST]
-			#
-			# Relevant SQLite documentation:
-			#   - https://www.sqlite.org/syntax/indexed-column.html
-			#   - https://www.sqlite.org/expridx.html
-			#
-			# the sortlist non-terminal stores a list of expression where each
-			# expression is optionally followed by ASC or DESC to indicate the
-			# sort order.
-			#
-			expr         = expression
-			direction_tk = maybe_one_of :ASC, :DESC
-			nulls_tk     = maybe_one_of [:NULLS, :FIRST].freeze, [:NULLS, :LAST].freeze
-
-			IndexedExpression.new(
-				expression: expr,
 				direction_tk: Token::Keyword(direction_tk),
-				nulls_tk: Token::Keyword(nulls_tk)
+				nulls_tk:     Token::Keyword(nulls_tk),
 			)
 		end
 
@@ -3590,7 +3603,7 @@ module Plume
 				direction = require current_token
 			end
 			if (nulls_kw = maybe(:NULLS))
-				nulls = require current_token
+				nulls = require_one_of(:FIRST, :LAST)
 			end
 
 			OrderingTerm.concrete(
@@ -3716,6 +3729,8 @@ module Plume
 				start_buffer = @peek_buffer.dup
 				start_lexer_cursor = @lexer.cursor
 				start_lexer_pos = @lexer.anchor
+				start_last_consumed = @last_consumed_token
+				start_last_consumed_fin = @last_consumed_fin
 
 				result = catch(:ERROR) { yield }
 
@@ -3723,6 +3738,8 @@ module Plume
 					@peek_buffer = start_buffer
 					@lexer.cursor = start_lexer_cursor
 					@lexer.anchor = start_lexer_pos
+					@last_consumed_token = start_last_consumed
+					@last_consumed_fin = start_last_consumed_fin
 					nil
 				else
 					result
@@ -3826,7 +3843,7 @@ module Plume
 			end
 			a << insert
 
-			collect(into: a, trailing_sep: trailing_sep, leading_sep: leading_sep, &)
+			collect(into: a, trailing_sep: trailing_sep, leading_sep: leading_sep, last_sep: sep_tk, &)
 
 			return token_span_from(*a) if a.all?(Array)
 
@@ -3865,7 +3882,7 @@ module Plume
 			if insert
 				a << insert
 
-				collect(into: a, trailing_sep: trailing_sep, leading_sep: leading_sep, &)
+				collect(into: a, trailing_sep: trailing_sep, leading_sep: leading_sep, last_sep: sep_tk, &)
 
 				return token_span_from(*a) if a.all?(Array)
 
@@ -3875,10 +3892,11 @@ module Plume
 			end
 		end
 
-		def collect(into: [], trailing_sep: nil, leading_sep: nil, &block)
-			break_next = false
+		# `last_sep` is what followed the last item in `into`: nil when no separator did, so the list has
+		# ended; a separator token, so another item must follow; false when an optional one was left out
+		def collect(into: [], trailing_sep: nil, leading_sep: nil, last_sep: false, &block)
 			while true
-				break if break_next
+				break if trailing_sep && last_sep.nil?
 
 				if leading_sep
 					sep_tk = (Array === leading_sep) ? maybe_one_of(*leading_sep) : maybe(leading_sep)
@@ -3894,11 +3912,10 @@ module Plume
 					end
 
 				elsif trailing_sep
-					val = maybe { yield }
+					val = last_sep ? yield : maybe { yield }
 					break if val.nil?
 
-					sep_tk = (Array === trailing_sep) ? maybe_one_of(*trailing_sep) : maybe(trailing_sep)
-					break_next = sep_tk.nil?
+					last_sep = sep_tk = (Array === trailing_sep) ? maybe_one_of(*trailing_sep) : maybe(trailing_sep)
 
 					insert = if Node === val
 						val.trailing = Token::Punctuation(sep_tk) if sep_tk
@@ -3993,6 +4010,7 @@ module Plume
 			ensure_buffer
 			token = @peek_buffer.shift
 			@last_consumed_fin = token[2]
+			@last_consumed_token = token[0]
 			token
 		end
 
@@ -4002,11 +4020,42 @@ module Plume
 			lexer = @lexer
 			@peek_buffer.tap do |buf|
 				while buf.size < size
-					buf << [lexer.next_token, lexer.anchor, lexer.cursor].freeze
+					token = lexer.next_token
+					if token == :WINDOW || token == :OVER || token == :FILTER
+						token = contextual_keyword(token, buf.empty? ? @last_consumed_token : buf.last[0])
+					end
+					buf << [token, lexer.anchor, lexer.cursor].freeze
 				end
 			end
 		end
 
+		# Like SQLite's tokenize.c, decide from the tokens around them whether WINDOW, OVER and FILTER are
+		# keywords here or names: WINDOW only before `name AS`, OVER only after `)` and before `(` or a
+		# name, FILTER only after `)` and before `(`
+		def contextual_keyword(token, previous)
+			following, after = lex_ahead
+			keyword =
+				case token
+				when :WINDOW then name_token?(following) && after == :AS
+				when :OVER then previous == :RP && (following == :LP || name_token?(following))
+				when :FILTER then previous == :RP && following == :LP
+				end
+			keyword ? token : :ID
+		end
+
+		# The types of the two tokens after the one just lexed, leaving the lexer where it was
+		def lex_ahead
+			cursor, anchor = @lexer.cursor, @lexer.anchor
+			[@lexer.next_token, @lexer.next_token]
+		ensure
+			@lexer.cursor, @lexer.anchor = cursor, anchor
+		end
+
+		# What SQLite's tokenize.c counts as a name when looking around WINDOW and OVER
+		def name_token?(token)
+			(token in :ID | :STRING | :WINDOW | :OVER) || JOIN_KEYWORDS.include?(token) || TOKEN_FALLBACKS.include?(token)
+		end
+
 		def expected!(*expected)
 			token, beg, fin = ensure_buffer[0]
 			throw :ERROR, ErrorMessage.new(@lexer.sql, token, beg, fin, expected)