graphql-c_parser 1.0.2 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5038dfe6e072d86fbd1e18e57bb7616374f4f37443ec4f0e7414a9cf514160d0
-  data.tar.gz: 9c3d7d016eda6be1fba8ab054ebe6e1163271cbadb87b1f6c4e86a0560a97d00
+  metadata.gz: 5978a06cdccd670bba134428979ff4ee120a88770256acbdb5c743660b277572
+  data.tar.gz: 455663b15baaf8752eb68ff9ef4d97e8ac51b2845c5805c327a733deb9b22881
 SHA512:
-  metadata.gz: cf3a0578d705b73ddb3a482c7fb92aecee25a11757d6d914b3d402c169dae8ebc251a1130a43e26451dc583d6d3537e1252c73c1523a24052847443542aa882e
-  data.tar.gz: 573d534232d1acf2eb8c5f0011dbed98b56d263480969dac57a342d702a594d9a826cd37dcd6527e810678129ae9c7594783e1d5b8a378445b5f070fc95fb777
+  metadata.gz: 6c9f90934f0e421f6cb17e33f12c085c1eda6ec070ae59591e1c106a1fa80688687c8f03349b688208fc6a2ac90d598697ba7d30a5311ed329bbe56125a90f1a
+  data.tar.gz: 4b90a2c70d1c4b36969f410dc658a2482f2897a0a07e03eda1c777a0867db6fc8b91e047dfa4b92edd4869005e5480ed736d3142bfb10ef0d10ccc8794025f9b

data/ext/graphql_c_parser_ext/graphql_c_parser_ext.c

@@ -1,6 +1,6 @@
 #include "graphql_c_parser_ext.h"
 
-VALUE GraphQL_CParser_Lexer_tokenize(VALUE self, VALUE query_string) {
+VALUE GraphQL_CParser_Lexer_tokenize_with_c(VALUE self, VALUE query_string) {
   return tokenize(query_string);
 }
 
@@ -13,7 +13,7 @@ void Init_graphql_c_parser_ext() {
   VALUE GraphQL = rb_define_module("GraphQL");
   VALUE CParser = rb_define_module_under(GraphQL, "CParser");
   VALUE Lexer = rb_define_module_under(CParser, "Lexer");
-  rb_define_singleton_method(Lexer, "tokenize", GraphQL_CParser_Lexer_tokenize, 1);
+  rb_define_singleton_method(Lexer, "tokenize_with_c", GraphQL_CParser_Lexer_tokenize_with_c, 1);
   setup_static_token_variables();
 
   VALUE Parser = rb_define_class_under(CParser, "Parser", rb_cObject);

data/ext/graphql_c_parser_ext/lexer.c

@@ -769,7 +769,8 @@ typedef enum TokenType {
 	BLOCK_STRING,
 	QUOTED_STRING,
 	UNKNOWN_CHAR,
-	COMMENT
+	COMMENT,
+	BAD_UNICODE_ESCAPE
 } TokenType;
 
 typedef struct Meta {
@@ -867,7 +868,7 @@ void emit(TokenType tt, char *ts, char *te, Meta *meta) {
 		token_sym = ID2SYM(rb_intern("STRING"));
 		quotes_length = 3;
 		token_content = rb_utf8_str_new(ts + quotes_length, (te - ts - (2 * quotes_length)));
-		line_incr = FIX2INT(rb_funcall(token_content, rb_intern("count"), 1, rb_str_new_cstr("\n")));
+		line_incr = FIX2INT(rb_funcall(token_content, rb_intern("count"), 1, rb_utf8_str_new_cstr("\n")));
 		break;
 		case STRING:
 		// This is used only by the parser, this is never reached
@@ -885,6 +886,10 @@ void emit(TokenType tt, char *ts, char *te, Meta *meta) {
 				token_content = rb_funcall(mGraphQLLanguageBlockString, rb_intern("trim_whitespace"), 1, token_content);
 			}
 			
+			// The parser doesn't distinguish between these,
+			// Maybe updated below if it's invalid UTF-8
+			tt = STRING;
+			
 			if (
 				RB_TEST(rb_funcall(token_content, rb_intern("valid_encoding?"), 0)) &&
 			RB_TEST(rb_funcall(token_content, rb_intern("match?"), 1, valid_string_pattern))
@@ -892,14 +897,12 @@ void emit(TokenType tt, char *ts, char *te, Meta *meta) {
 				rb_funcall(mGraphQLLanguageLexer, rb_intern("replace_escaped_characters_in_place"), 1, token_content);
 				if (!RB_TEST(rb_funcall(token_content, rb_intern("valid_encoding?"), 0))) {
 					token_sym = ID2SYM(rb_intern("BAD_UNICODE_ESCAPE"));
+					tt = BAD_UNICODE_ESCAPE;
 				}
-				
-				
 			} else {
 				token_sym = ID2SYM(rb_intern("BAD_UNICODE_ESCAPE"));
+				tt = BAD_UNICODE_ESCAPE;
 			}
-			// The parser doesn't distinguish between these:
-			tt = STRING;
 		}
 		
 		VALUE token = rb_ary_new_from_args(6,
@@ -935,7 +938,7 @@ VALUE tokenize(VALUE query_rbstr) {
 	Meta *meta = &meta_s;
 	
 	
-#line 939 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 942 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 	{
 		cs = (int)graphql_c_lexer_start;
 		ts = 0;
@@ -943,10 +946,10 @@ VALUE tokenize(VALUE query_rbstr) {
 		act = 0;
 	}
 	
-#line 351 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
+#line 354 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 	
 	
-#line 950 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 953 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 	{
 		unsigned int _trans = 0;
 		const char * _keys;
@@ -961,7 +964,7 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 					{ts = p;}}
 				
-#line 965 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 968 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 				
 				
 				break; 
@@ -999,7 +1002,7 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1003 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1006 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1012,7 +1015,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(RCURLY, ts, te, meta); }
 						}}
 					
-#line 1016 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1019 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1025,7 +1028,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(LCURLY, ts, te, meta); }
 						}}
 					
-#line 1029 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1032 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1038,7 +1041,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(RPAREN, ts, te, meta); }
 						}}
 					
-#line 1042 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1045 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1051,7 +1054,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(LPAREN, ts, te, meta); }
 						}}
 					
-#line 1055 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1058 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1064,7 +1067,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(RBRACKET, ts, te, meta); }
 						}}
 					
-#line 1068 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1071 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1077,7 +1080,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(LBRACKET, ts, te, meta); }
 						}}
 					
-#line 1081 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1084 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1090,7 +1093,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(COLON, ts, te, meta); }
 						}}
 					
-#line 1094 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1097 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1103,7 +1106,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(BLOCK_STRING, ts, te, meta); }
 						}}
 					
-#line 1107 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1110 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1116,7 +1119,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(QUOTED_STRING, ts, te, meta); }
 						}}
 					
-#line 1120 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1123 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1129,7 +1132,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(VAR_SIGN, ts, te, meta); }
 						}}
 					
-#line 1133 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1136 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1142,7 +1145,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(DIR_SIGN, ts, te, meta); }
 						}}
 					
-#line 1146 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1149 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1155,7 +1158,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(ELLIPSIS, ts, te, meta); }
 						}}
 					
-#line 1159 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1162 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1168,7 +1171,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(EQUALS, ts, te, meta); }
 						}}
 					
-#line 1172 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1175 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1181,7 +1184,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(BANG, ts, te, meta); }
 						}}
 					
-#line 1185 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1188 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1194,7 +1197,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(PIPE, ts, te, meta); }
 						}}
 					
-#line 1198 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1201 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1207,7 +1210,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(AMP, ts, te, meta); }
 						}}
 					
-#line 1211 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1214 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1223,7 +1226,7 @@ VALUE tokenize(VALUE query_rbstr) {
 							}
 						}}
 					
-#line 1227 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1230 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1236,7 +1239,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(UNKNOWN_CHAR, ts, te, meta); }
 						}}
 					
-#line 1240 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1243 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1249,7 +1252,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(INT, ts, te, meta); }
 						}}
 					
-#line 1253 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1256 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1262,7 +1265,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(FLOAT, ts, te, meta); }
 						}}
 					
-#line 1266 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1269 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1275,7 +1278,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(BLOCK_STRING, ts, te, meta); }
 						}}
 					
-#line 1279 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1282 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1288,7 +1291,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(QUOTED_STRING, ts, te, meta); }
 						}}
 					
-#line 1292 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1295 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1301,7 +1304,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(IDENTIFIER, ts, te, meta); }
 						}}
 					
-#line 1305 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1308 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1314,7 +1317,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(COMMENT, ts, te, meta); }
 						}}
 					
-#line 1318 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1321 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1327,7 +1330,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								meta->col += te - ts; }
 						}}
 					
-#line 1331 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1334 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1340,7 +1343,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(UNKNOWN_CHAR, ts, te, meta); }
 						}}
 					
-#line 1344 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1347 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1354,7 +1357,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(INT, ts, te, meta); }
 						}}
 					
-#line 1358 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1361 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1368,7 +1371,7 @@ VALUE tokenize(VALUE query_rbstr) {
 								emit(UNKNOWN_CHAR, ts, te, meta); }
 						}}
 					
-#line 1372 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1375 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1548,7 +1551,7 @@ VALUE tokenize(VALUE query_rbstr) {
 							}}
 					}
 					
-#line 1552 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1555 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1558,13 +1561,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1562 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1565 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 54 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 1;}}
 					
-#line 1568 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1571 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1574,13 +1577,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1578 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1581 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 55 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 2;}}
 					
-#line 1584 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1587 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1590,13 +1593,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1594 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1597 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 56 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 3;}}
 					
-#line 1600 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1603 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1606,13 +1609,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1610 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1613 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 57 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 4;}}
 					
-#line 1616 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1619 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1622,13 +1625,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1626 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1629 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 58 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 5;}}
 					
-#line 1632 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1635 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1638,13 +1641,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1642 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1645 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 59 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 6;}}
 					
-#line 1648 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1651 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1654,13 +1657,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1658 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1661 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 60 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 7;}}
 					
-#line 1664 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1667 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1670,13 +1673,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1674 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1677 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 61 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 8;}}
 					
-#line 1680 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1683 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1686,13 +1689,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1690 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1693 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 62 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 9;}}
 					
-#line 1696 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1699 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1702,13 +1705,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1706 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1709 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 63 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 10;}}
 					
-#line 1712 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1715 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1718,13 +1721,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1722 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1725 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 64 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 11;}}
 					
-#line 1728 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1731 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1734,13 +1737,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1738 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1741 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 65 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 12;}}
 					
-#line 1744 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1747 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1750,13 +1753,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1754 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1757 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 66 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 13;}}
 					
-#line 1760 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1763 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1766,13 +1769,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1770 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1773 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 67 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 14;}}
 					
-#line 1776 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1779 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1782,13 +1785,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1786 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1789 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 68 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 15;}}
 					
-#line 1792 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1795 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1798,13 +1801,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1802 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1805 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 69 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 16;}}
 					
-#line 1808 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1811 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1814,13 +1817,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1818 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1821 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 70 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 17;}}
 					
-#line 1824 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1827 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1830,13 +1833,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1834 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1837 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 71 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 18;}}
 					
-#line 1840 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1843 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1846,13 +1849,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1850 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1853 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 72 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 19;}}
 					
-#line 1856 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1859 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1862,13 +1865,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1866 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1869 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 73 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 20;}}
 					
-#line 1872 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1875 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1878,13 +1881,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1882 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1885 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 74 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 21;}}
 					
-#line 1888 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1891 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1894,13 +1897,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1898 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1901 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 82 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 29;}}
 					
-#line 1904 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1907 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1910,13 +1913,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1914 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1917 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 83 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 30;}}
 					
-#line 1920 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1923 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1926,13 +1929,13 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{te = p+1;}}
 					
-#line 1930 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1933 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					{
 #line 91 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 						{act = 38;}}
 					
-#line 1936 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1939 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1952,7 +1955,7 @@ VALUE tokenize(VALUE query_rbstr) {
 #line 1 "NONE"
 						{ts = 0;}}
 					
-#line 1956 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
+#line 1959 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.c"
 					
 					
 					break; 
@@ -1965,7 +1968,7 @@ VALUE tokenize(VALUE query_rbstr) {
 		_out: {}
 	}
 	
-#line 352 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
+#line 355 "graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl"
 	
 	
 	return tokens;
@@ -1973,12 +1976,12 @@ VALUE tokenize(VALUE query_rbstr) {
 
 
 #define SETUP_STATIC_TOKEN_VARIABLE(token_name, token_content) \
-GraphQLTokenString##token_name = rb_str_new_cstr(token_content); \
+GraphQLTokenString##token_name = rb_utf8_str_new_cstr(token_content); \
 rb_funcall(GraphQLTokenString##token_name, rb_intern("-@"), 0); \
 rb_global_variable(&GraphQLTokenString##token_name); \
 
 #define SETUP_STATIC_STRING(var_name, str_content) \
-var_name = rb_str_new_cstr(str_content); \
+var_name = rb_utf8_str_new_cstr(str_content); \
 rb_global_variable(&var_name); \
 rb_str_freeze(var_name); \
 

data/ext/graphql_c_parser_ext/parser.c

@@ -3233,7 +3233,8 @@ yyreturnlab:
 
 // Custom functions
 int yylex (YYSTYPE *lvalp, VALUE parser, VALUE filename) {
-  int next_token_idx = FIX2INT(rb_ivar_get(parser, rb_intern("@next_token_index")));
+  VALUE next_token_idx_rb_int = rb_ivar_get(parser, rb_intern("@next_token_index"));
+  int next_token_idx = FIX2INT(next_token_idx_rb_int);
   VALUE tokens = rb_ivar_get(parser, rb_intern("@tokens"));
   VALUE next_token = rb_ary_entry(tokens, next_token_idx);
 
@@ -3243,7 +3244,15 @@ int yylex (YYSTYPE *lvalp, VALUE parser, VALUE filename) {
   rb_ivar_set(parser, rb_intern("@next_token_index"), INT2FIX(next_token_idx + 1));
   VALUE token_type_rb_int = rb_ary_entry(next_token, 5);
   int next_token_type = FIX2INT(token_type_rb_int);
-
+  if (next_token_type == 241) { // BAD_UNICODE_ESCAPE
+    VALUE mGraphQL = rb_const_get_at(rb_cObject, rb_intern("GraphQL"));
+    VALUE mCParser = rb_const_get_at(mGraphQL, rb_intern("CParser"));
+    VALUE bad_unicode_error = rb_funcall(
+        mCParser, rb_intern("prepare_bad_unicode_error"), 1,
+        parser
+    );
+    rb_exc_raise(bad_unicode_error);
+  }
   *lvalp = next_token;
   return next_token_type;
 }

data/lib/graphql/c_parser/version.rb

@@ -2,6 +2,6 @@
 
 module GraphQL
   module CParser
-    VERSION = "1.0.2"
+    VERSION = "1.0.4"
   end
 end

data/lib/graphql/c_parser.rb

@@ -40,6 +40,40 @@ module GraphQL
       GraphQL::ParseError.new(message, line, col, parser.query_string, filename: parser.filename)
     end
 
+    def self.prepare_bad_unicode_error(parser)
+      token = parser.tokens[parser.next_token_index - 1]
+      line = token[1]
+      col = token[2]
+      GraphQL::ParseError.new(
+        "Parse error on bad Unicode escape sequence: #{token[3].inspect} (error) at [#{line}, #{col}]",
+        line,
+        col,
+        parser.query_string,
+        filename: parser.filename
+      )
+    end
+
+    module Lexer
+      def self.tokenize(graphql_string)
+        if !(graphql_string.encoding == Encoding::UTF_8 || graphql_string.ascii_only?)
+          graphql_string = graphql_string.dup.force_encoding(Encoding::UTF_8)
+        end
+        if !graphql_string.valid_encoding?
+          return [
+            [
+              :BAD_UNICODE_ESCAPE,
+              1,
+              1,
+              graphql_string,
+              nil, # prev token
+              241 # BAD_UNICODE_ESCAPE in lexer.rl
+            ]
+          ]
+        end
+        tokenize_with_c(graphql_string)
+      end
+    end
+
     class Parser
       def initialize(query_string, filename, trace)
         if query_string.nil?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: graphql-c_parser
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.4
 platform: ruby
 authors:
 - Robert Mosolgo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-12 00:00:00.000000000 Z
+date: 2023-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: graphql