@surrealdb/lezer 1.0.0-beta.8 → 1.0.0

package/src/tokens.js

@@ -1,4 +1,4 @@
-import {ExternalTokenizer} from "@lezer/lr";
+import { ExternalTokenizer } from "@lezer/lr";
 
 import {
 	_break,
@@ -15,28 +15,40 @@ import {
 	_with,
 	access,
 	algorithm,
+	all,
 	alter,
+	always,
 	analyzer,
 	any,
+	api,
 	as,
 	asc,
 	assert,
 	at,
 	authenticate,
+	auto,
+	backend,
 	begin,
 	bm25,
+	bucket,
 	by,
 	cancel,
 	capacity,
+	cascade,
 	changefeed,
 	changes,
+	collate,
 	columns,
 	comment,
 	commit,
+	computed,
+	concurrently,
+	config,
 	content,
 	create,
 	database,
 	db,
+	defer,
 	define,
 	desc,
 	dimension,
@@ -50,9 +62,12 @@ import {
 	duration,
 	efc,
 	end,
+	enforced,
 	event,
+	exclude,
 	exists,
 	explain,
+	expunge,
 	extend_candidates,
 	fetch,
 	field,
@@ -60,10 +75,14 @@ import {
 	filters,
 	flexible,
 	from,
+	functions,
+	get,
+	graphql,
 	group,
 	highlights,
 	hnsw,
 	ignore,
+	include,
 	index,
 	info,
 	insert,
@@ -79,6 +98,7 @@ import {
 	m,
 	m0,
 	merge,
+	middleware,
 	mtree_cache,
 	mtree,
 	namespace,
@@ -86,6 +106,8 @@ import {
 	normal,
 	not,
 	ns,
+	numeric,
+	omit,
 	on,
 	only,
 	option,
@@ -98,14 +120,19 @@ import {
 	password,
 	patch,
 	permissions,
+	post,
 	postings_cache,
 	postings_order,
+	put,
 	readonly,
 	rebuild,
 	record,
+	reference,
+	reject,
 	relate,
 	relation,
 	remove,
+	replace,
 	roles,
 	root,
 	sc,
@@ -119,12 +146,13 @@ import {
 	show,
 	signin,
 	signup,
-	since,
+	since as _since,
 	sleep,
 	split,
 	start,
 	structure,
 	table,
+	tables,
 	tb,
 	tempfiles,
 	terms_cache,
@@ -134,6 +162,7 @@ import {
 	to,
 	token,
 	tokenizers,
+	trace,
 	transaction,
 	typeKeyword,
 	unique,
@@ -145,6 +174,7 @@ import {
 	user,
 	valueKeyword,
 	values,
+	version,
 	when,
 	where,
 
@@ -157,103 +187,67 @@ import {
 	diff,
 	full,
 	none,
-
-	f32,
-	f64,
-	i16,
-	i32,
-	i64,
-
-	createPermissions,
-	deletePermissions,
-	selectPermissions,
-	updatePermissions,
-
-	jwks,
-	eddsa,
-	es256,
-	es384,
-	es512,
-	ps256,
-	ps384,
-	ps512,
-	rs256,
-	rs384,
-	rs512,
-
-	allinside,
-	and,
-	anyinside,
-	contains,
-	containsall,
-	containsany,
-	containsnone,
-	containsnot,
-	inside,
-	intersects,
+	IndexTypeClause,
+	TokenType,
 	is,
-	noneinside,
-	notinside,
+	binaryOperatorKeyword,
 	opIn,
 	opNot,
-	or,
-	outside,
-
-	chebyshev,
-	cosine,
-	euclidean,
-	hamming,
-	jaccard,
-	manhattan,
+	Distance,
 	minkowski,
-	pearson,
-
-	ascii,
-	edgengram,
-	ngram,
-	snowball,
-	uppercase,
-
-	_class,
-	blank,
-	camel,
-	punct,
-
+	Filter,
+	AnalyzerTokenizer,
 	_function,
 	rand,
 	count,
-
 	objectOpen,
+
+	rangeOp,
+	rangeOpOpenLeft,
+	rangeOpOpenRight,
+	rangeOpOpenBoth
 } from "./parser.terms";
 
 const tokenMap = {
 	access,
 	algorithm,
+	all,
 	alter,
+	always,
 	analyzer,
 	any,
+	api,
 	as,
 	asc,
 	assert,
 	at,
 	authenticate,
+	auto,
+	backend,
 	begin,
 	bm25,
 	break: _break,
+	bucket,
 	by,
 	cancel,
 	capacity,
+	cascade,
 	changefeed,
 	changes,
+	collate,
 	columns,
 	comment,
 	commit,
+	computed,
+	concurrently,
+	config,
 	content,
 	continue: _continue,
 	create,
 	database,
 	db,
 	default: _default,
+	defer,
 	define,
 	delete: _delete,
 	desc,
@@ -269,9 +263,12 @@ const tokenMap = {
 	efc,
 	else: _else,
 	end,
+	enforced,
 	event,
+	exclude,
 	exists,
 	explain,
+	expunge,
 	extend_candidates,
 	fetch,
 	field,
@@ -280,12 +277,16 @@ const tokenMap = {
 	flexible,
 	for: _for,
 	from,
+	functions,
+	get,
+	graphql,
 	group,
 	highlights,
 	hnsw,
 	if: _if,
 	ignore,
 	in: _in,
+	include,
 	index,
 	info,
 	insert,
@@ -302,6 +303,7 @@ const tokenMap = {
 	m,
 	m0,
 	merge,
+	middleware,
 	mtree_cache,
 	mtree,
 	namespace,
@@ -309,6 +311,8 @@ const tokenMap = {
 	normal,
 	not,
 	ns,
+	numeric,
+	omit,
 	on,
 	only,
 	option,
@@ -321,14 +325,19 @@ const tokenMap = {
 	password,
 	patch,
 	permissions,
+	post,
 	postings_cache,
 	postings_order,
+	put,
 	readonly,
 	rebuild,
 	record,
+	reference,
+	reject,
 	relate,
 	relation,
 	remove,
+	replace,
 	return: _return,
 	roles,
 	root,
@@ -343,12 +352,13 @@ const tokenMap = {
 	show,
 	signin,
 	signup,
-	since,
+	since: _since,
 	sleep,
 	split,
 	start,
 	structure,
 	table,
+	tables,
 	tb,
 	tempfiles,
 	terms_cache,
@@ -359,6 +369,7 @@ const tokenMap = {
 	to,
 	token,
 	tokenizers,
+	trace,
 	transaction,
 	type: typeKeyword,
 	unique,
@@ -370,6 +381,7 @@ const tokenMap = {
 	user,
 	value: valueKeyword,
 	values,
+	version,
 	when,
 	where,
 	with: _with,
@@ -384,59 +396,58 @@ const tokenMap = {
 	null: _null,
 	true: _true,
 
-	f32,
-	f64,
-	i16,
-	i32,
-	i64,
+	f32: IndexTypeClause,
+	f64: IndexTypeClause,
+	i16: IndexTypeClause,
+	i32: IndexTypeClause,
+	i64: IndexTypeClause,
 
-	jwks,
-	eddsa,
-	es256,
-	es384,
-	es512,
-	ps256,
-	ps384,
-	ps512,
-	rs256,
-	rs384,
-	rs512,
+	jwks: TokenType,
+	eddsa: TokenType,
+	es256: TokenType,
+	es384: TokenType,
+	es512: TokenType,
+	ps256: TokenType,
+	ps384: TokenType,
+	ps512: TokenType,
+	rs256: TokenType,
+	rs384: TokenType,
+	rs512: TokenType,
 
-	allinside,
-	and,
-	anyinside,
-	contains,
-	containsall,
-	containsany,
-	containsnone,
-	containsnot,
-	inside,
-	intersects,
+	and: binaryOperatorKeyword,
+	or: binaryOperatorKeyword,
+	contains: binaryOperatorKeyword,
+	containsnot: binaryOperatorKeyword,
+	containsall: binaryOperatorKeyword,
+	containsany: binaryOperatorKeyword,
+	containsnone: binaryOperatorKeyword,
+	inside: binaryOperatorKeyword, notinside: binaryOperatorKeyword,
+	allinside: binaryOperatorKeyword,
+	anyinside: binaryOperatorKeyword,
+	noneinside: binaryOperatorKeyword,
+	outside: binaryOperatorKeyword,
+	intersects: binaryOperatorKeyword,
 	is,
-	noneinside,
-	notinside,
-	or,
-	outside,
 
-	chebyshev,
-	cosine,
-	euclidean,
-	hamming,
-	jaccard,
-	manhattan,
+	chebyshev: Distance,
+	cosine: Distance,
+	euclidean: Distance,
+	hamming: Distance,
+	jaccard: Distance,
+	manhattan: Distance,
 	minkowski,
-	pearson,
+	pearson: Distance,
 
-	ascii,
-	edgengram,
-	ngram,
-	snowball,
-	uppercase,
+	ascii: Filter,
+	edgengram: Filter,
+	ngram: Filter,
+	snowball: Filter,
+	uppercase: Filter,
 
-	blank,
-	camel,
-	class: _class,
-	punct,
+	blank: AnalyzerTokenizer,
+	camel: AnalyzerTokenizer,
+	class: AnalyzerTokenizer,
+	punct: AnalyzerTokenizer,
 
 	// Function names
 	function: _function,
@@ -444,32 +455,26 @@ const tokenMap = {
 	count,
 };
 
-const tryMapped = new Map([
-	["select", [selectPermissions]],
-	["create", [createPermissions]],
-	["update", [updatePermissions]],
-	["delete", [deletePermissions]],
-	["not", [opNot]],
-	["in", [opIn]],
-]);
-
-export const tokens = function(t, stack) {
-	for (const tk of tryMapped.get(t.toLowerCase()) ?? []) {
-		if (stack.canShift(tk)) return tk;
-	}
-
+export const tokens = function (t, stack) {
 	return tokenMap[t.toLowerCase()] ?? -1;
+};
+
+function isSpace(ch) {
+	return ch === 32 || ch === 9 || ch === 10 || ch === 13
 }
 
 function skipSpace(input, off) {
-	for (;;) {
+	for (; ;) {
 		let next = input.peek(off);
-		if (next === 32 || next === 9 || next === 10 || next === 13) {
+		if (isSpace(next)) {
 			off++;
-		} else if (next === 35 /* '#' */ ||
-				   (next === 47 /* '/' */ || next === 45 /* '-' */) && input.peek(off + 1) === next) {
+		} else if (
+			next === 35 /* '#' */ ||
+			((next === 47 /* '/' */ || next === 45) /* '-' */ &&
+				input.peek(off + 1) === next)
+		) {
 			off++;
-			for (;;) {
+			for (; ;) {
 				let next = input.peek(off);
 				if (next < 0 || next === 10 || next === 13) break;
 				off++;
@@ -481,7 +486,12 @@ function skipSpace(input, off) {
 }
 
 function isIdentifierChar(ch) {
-	return ch === 95 || ch >= 65 && ch <= 90 || ch >= 97 && ch <= 122 || ch >= 48 && ch <= 57;
+	return (
+		ch === 95 ||
+		(ch >= 65 && ch <= 90) ||
+		(ch >= 97 && ch <= 122) ||
+		(ch >= 48 && ch <= 57)
+	);
 }
 
 function skipObjKey(input, off) {
@@ -491,17 +501,19 @@ function skipObjKey(input, off) {
 			off++;
 		} while (isIdentifierChar(input.peek(off)));
 		return off;
-	} else if (first === 38 /* "'" */ || first === 34 /* '"' */) {
-		for (let escaped = false;;) {
+	} else if (first === 39 /* "'" */ || first === 34 /* '"' */) {
+		for (let escaped = false; ;) {
 			let next = input.peek(++off);
 			if (next < 0) return off;
 			if (next === first && !escaped) return off + 1;
-			escaped = next === 92 /* '\\' */
+			escaped = next === 92; /* '\\' */
 		}
 	}
+
+	return null;
 }
 
-export const objectToken = new ExternalTokenizer((input, _stack) => {
+export const objectToken = new ExternalTokenizer(input => {
 	if (input.next === 123 /* '{' */) {
 		let off = skipSpace(input, 1);
 
@@ -530,3 +542,30 @@ export const objectToken = new ExternalTokenizer((input, _stack) => {
 		}
 	}
 });
+
+function closedRangeBefore(ch) {
+	return isSpace(ch) || ch < 0 || ch === 91 /* '[' */ || ch === 44 /* ',' */ ||
+		ch === 123 /* '{' */ || ch === 40 /* '(' */ || ch === 59 /* ';' */ || ch === 58 /* ':' */
+}
+
+function closedRangeAfter(ch) {
+	return isSpace(ch) || ch < 0 || ch === 93 /* ']' */ || ch === 44 /* ',' */ ||
+		ch === 125 /* '}' */ || ch === 41 /* ')' */ || ch === 59 /* ';' */ || ch === 58 /* ':' */
+}
+
+export const rangeOperator = new ExternalTokenizer(input => {
+	if (input.next === 46 /* '.' */ && input.peek(1) === 46 ||
+		input.next === 62 /* '>' */ && input.peek(1) === 46 && input.peek(1) === 46) {
+		let inclStart = input.next !== 62
+		let closedBefore = closedRangeBefore(input.peek(-1))
+		if (!inclStart && closedBefore) return
+		input.advance(inclStart ? 2 : 3)
+		let inclEnd = input.next === 61 /* '=' */
+		if (inclEnd) input.advance()
+		let closedAfter = closedRangeAfter(input.next) || input.next < 0
+		if (inclEnd && closedAfter) return
+		input.acceptToken(closedBefore && closedAfter ? rangeOpOpenBoth
+			: closedBefore ? rangeOpOpenLeft
+				: closedAfter ? rangeOpOpenRight : rangeOp)
+	}
+})

package/src/typing.d.ts

@@ -0,0 +1,6 @@
+import type { NodeProp } from "@lezer/common";
+import type { LRParser } from "@lezer/lr";
+
+export const parser: LRParser;
+export const sinceProp: NodeProp<string>;
+export const untilProp: NodeProp<string>;

package/src/version.js

@@ -0,0 +1,15 @@
+import { NodeProp } from '@lezer/common';
+
+export const since = new NodeProp({
+	perNode: false,
+	deserialize(str) {
+		return str.replace(/_/g, '.').toLowerCase();
+	},
+});
+
+export const until = new NodeProp({
+	perNode: false,
+	deserialize(str) {
+		return str.replace(/_/g, '.').toLowerCase();
+	},
+});

package/test/statement.txt

@@ -45,3 +45,88 @@ SurrealQL(
 	LiveSelectStatement(Keyword,Keyword,Any,Keyword,Ident),
 	LiveSelectStatement(Keyword,Keyword,Any,Keyword,RecordId(RecordTbIdent,Colon,RecordIdIdent)),
 )
+
+# Let Substatements
+
+LET $bla = (SELECT * FROM 123);
+LET $bla = SELECT * FROM 123;
+
+==>
+
+SurrealQL(
+  LetStatement(Keyword,VariableName,SubQuery(SelectStatement(Keyword,Any,Keyword,Int))),
+  LetStatement(Keyword,VariableName,SelectStatement(Keyword,Any,Keyword,Int)))
+
+# Block Comments
+
+1 /* this is
+     a block
+     comment */ + 2;
+
+==>
+
+SurrealQL(BinaryExpression(Int,BlockComment,Operator,Int))
+
+# Version Clause
+
+SELECT * FROM person VERSION d'2024-10-10T14:37:08.444Z'
+
+==>
+
+SurrealQL(SelectStatement(Keyword,Any,Keyword,Ident,VersionClause(Keyword,String)))
+
+
+# Recurse part
+
+person:tobie.{..}->knows->person;
+person:tobie.{..}(->knows->person).name;
+person:tobie.{..}.{ id, name, knows: ->knows->person.@ };
+a:1.{1};
+a:1.{1..};
+a:1.{1..2};
+a:1.{..2};
+@.name;
+@name;
+
+==>
+
+SurrealQL(
+  Path(
+    RecordId(RecordTbIdent,Colon,RecordIdIdent),
+    Subscript(Recurse(BraceOpen,RecurseRange(RangeOp),BraceClose)),
+    GraphPath(ArrowRight,Ident), GraphPath(ArrowRight,Ident)
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,RecordIdIdent),
+    Subscript(
+      Recurse(
+        BraceOpen, RecurseRange(RangeOp), BraceClose,
+        Path(GraphPath(ArrowRight,Ident), GraphPath(ArrowRight,Ident))
+      )
+    ),
+    Subscript(Ident)
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,RecordIdIdent),
+    Subscript(Recurse(BraceOpen,RecurseRange(RangeOp),BraceClose)),
+    Subscript(Destructure(BraceOpen,Ident,Ident,Ident,Colon,GraphPath(ArrowRight,Ident),GraphPath(ArrowRight,Ident),Subscript(At),BraceClose))
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,Int),
+    Subscript(Recurse(BraceOpen,RecurseRange(Int),BraceClose))
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,Int),
+    Subscript(Recurse(BraceOpen,RecurseRange(Int,RangeOp),BraceClose))
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,Int),
+    Subscript(Recurse(BraceOpen,RecurseRange(Int,RangeOp,Int),BraceClose))
+  ),
+  Path(
+    RecordId(RecordTbIdent,Colon,Int),
+    Subscript(Recurse(BraceOpen,RecurseRange(RangeOp,Int),BraceClose))
+  ),
+  Path(At,Subscript(Ident)),
+  Path(At,Ident)
+)

package/test/test-surrealql.js

@@ -1,9 +1,9 @@
-import {parser} from "../dist/index.js";
-import {fileTests} from "@lezer/generator/dist/test";
+import { parser } from "../dist/index.js";
+import { fileTests } from "@lezer/generator/dist/test";
 
 import * as fs from "fs";
 import * as path from "path";
-import {fileURLToPath} from "url";
+import { fileURLToPath } from "url";
 let caseDir = path.dirname(fileURLToPath(import.meta.url));
 
 let filter = process.argv[2];
@@ -11,12 +11,12 @@ let filter = process.argv[2];
 for (let file of fs.readdirSync(caseDir)) {
 	if (!/\.txt$/.test(file)) continue;
 	console.log("File " + file + ":");
-	for (let {name, run} of fileTests(fs.readFileSync(path.join(caseDir, file), "utf8"), file)) {
+	for (let { name, run } of fileTests(fs.readFileSync(path.join(caseDir, file), "utf8"), file)) {
 		if (!filter || name.indexOf(filter) > -1) {
 			try {
 				run(parser);
 				console.log(" ✔ " + name);
-			} catch(e) {
+			} catch (e) {
 				console.log(" ✘ " + name + "\n   " + String(e.message || e).replace(/\n/g, "\n   "))
 			}
 		}