prism 1.9.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +144 -40
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +49 -20
  6. data/README.md +38 -98
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +9 -16
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +39 -28
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1527 -1467
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +759 -418
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +45 -0
  23. data/include/prism/arena.hpp +42 -0
  24. data/include/prism/ast.h +2738 -611
  25. data/include/prism/ast.hpp +641 -0
  26. data/include/prism/buffer.h +60 -0
  27. data/include/prism/buffer.hpp +55 -0
  28. data/include/prism/comments.h +72 -0
  29. data/include/prism/compiler/accel.h +26 -0
  30. data/include/prism/compiler/align.h +36 -0
  31. data/include/prism/compiler/assume.h +25 -0
  32. data/include/prism/compiler/branch.h +26 -0
  33. data/include/prism/compiler/bswap.h +46 -0
  34. data/include/prism/compiler/cplusplus.hpp +16 -0
  35. data/include/prism/compiler/endianness.h +34 -0
  36. data/include/prism/compiler/exceptions.hpp +17 -0
  37. data/include/prism/compiler/exported.h +24 -0
  38. data/include/prism/compiler/fallthrough.h +22 -0
  39. data/include/prism/compiler/filesystem.h +32 -0
  40. data/include/prism/compiler/flex_array.h +19 -0
  41. data/include/prism/compiler/force_inline.h +21 -0
  42. data/include/prism/compiler/format.h +27 -0
  43. data/include/prism/compiler/inline.h +17 -0
  44. data/include/prism/compiler/lifetimebound.hpp +25 -0
  45. data/include/prism/compiler/nodiscard.h +22 -0
  46. data/include/prism/compiler/nonnull.h +18 -0
  47. data/include/prism/compiler/unused.h +18 -0
  48. data/include/prism/constant_pool.h +106 -0
  49. data/include/prism/diagnostic.h +55 -402
  50. data/include/prism/errors_format.h +61 -0
  51. data/include/prism/excludes.h +29 -0
  52. data/include/prism/integer.h +41 -0
  53. data/include/prism/internal/allocator.h +68 -0
  54. data/include/prism/internal/allocator_debug.h +88 -0
  55. data/include/prism/internal/arena.h +108 -0
  56. data/include/prism/internal/bit.h +42 -0
  57. data/include/prism/internal/buffer.h +94 -0
  58. data/include/prism/internal/char.h +145 -0
  59. data/include/prism/internal/comments.h +54 -0
  60. data/include/prism/internal/constant_pool.h +152 -0
  61. data/include/prism/internal/diagnostic.h +389 -0
  62. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  63. data/include/prism/internal/integer.h +68 -0
  64. data/include/prism/internal/isinf.h +16 -0
  65. data/include/prism/internal/line_offset_list.h +34 -0
  66. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  67. data/include/prism/internal/magic_comments.h +23 -0
  68. data/include/prism/internal/memchr.h +15 -0
  69. data/include/prism/internal/node.h +42 -0
  70. data/include/prism/internal/options.h +232 -0
  71. data/include/prism/internal/parser.h +1015 -0
  72. data/include/prism/internal/regexp.h +26 -0
  73. data/include/prism/internal/serialize.h +39 -0
  74. data/include/prism/internal/source.h +72 -0
  75. data/include/prism/internal/static_literals.h +60 -0
  76. data/include/prism/internal/stringy.h +30 -0
  77. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  78. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  79. data/include/prism/internal/tokens.h +11 -0
  80. data/include/prism/internal/xxh3.h +16 -0
  81. data/include/prism/json.h +40 -0
  82. data/include/prism/line_offset_list.h +69 -0
  83. data/include/prism/magic_comments.h +53 -0
  84. data/include/prism/node.h +59 -49
  85. data/include/prism/node.hpp +272 -0
  86. data/include/prism/node_type.hpp +38 -0
  87. data/include/prism/options.h +154 -279
  88. data/include/prism/options.hpp +43 -0
  89. data/include/prism/parser.h +367 -870
  90. data/include/prism/parser.hpp +274 -0
  91. data/include/prism/prettyprint.h +14 -9
  92. data/include/prism/serialize.h +123 -0
  93. data/include/prism/source.h +163 -0
  94. data/include/prism/stream.h +36 -0
  95. data/include/prism/string_query.h +82 -0
  96. data/include/prism/stringy.h +61 -0
  97. data/include/prism/stringy.hpp +31 -0
  98. data/include/prism/version.h +20 -3
  99. data/include/prism.h +35 -320
  100. data/include/prism.hpp +24 -0
  101. data/lib/prism/compiler.rb +319 -305
  102. data/lib/prism/desugar_compiler.rb +105 -34
  103. data/lib/prism/dispatcher.rb +646 -473
  104. data/lib/prism/dot_visitor.rb +592 -511
  105. data/lib/prism/dsl.rb +391 -25
  106. data/lib/prism/ffi.rb +258 -90
  107. data/lib/prism/inspect_visitor.rb +357 -325
  108. data/lib/prism/lex_compat.rb +254 -212
  109. data/lib/prism/mutation_compiler.rb +313 -305
  110. data/lib/prism/node.rb +20966 -6992
  111. data/lib/prism/node_ext.rb +180 -187
  112. data/lib/prism/node_find.rb +215 -0
  113. data/lib/prism/parse_result/errors.rb +7 -1
  114. data/lib/prism/parse_result/newlines.rb +276 -8
  115. data/lib/prism/parse_result.rb +468 -139
  116. data/lib/prism/pattern.rb +65 -20
  117. data/lib/prism/reflection.rb +23 -12
  118. data/lib/prism/relocation.rb +149 -87
  119. data/lib/prism/serialize.rb +2889 -700
  120. data/lib/prism/string_query.rb +24 -1
  121. data/lib/prism/translation/parser/builder.rb +10 -2
  122. data/lib/prism/translation/parser/compiler.rb +44 -52
  123. data/lib/prism/translation/parser/lexer.rb +41 -92
  124. data/lib/prism/translation/parser.rb +1 -1
  125. data/lib/prism/translation/parser_current.rb +1 -1
  126. data/lib/prism/translation/ripper/lexer.rb +8 -10
  127. data/lib/prism/translation/ripper/sexp.rb +2 -10
  128. data/lib/prism/translation/ripper/shim.rb +2 -0
  129. data/lib/prism/translation/ripper.rb +967 -209
  130. data/lib/prism/translation/ruby_parser.rb +73 -316
  131. data/lib/prism/translation.rb +2 -0
  132. data/lib/prism/visitor.rb +632 -5
  133. data/lib/prism.rb +79 -9
  134. data/prism.gemspec +141 -62
  135. data/rbi/generated/prism/compiler.rbi +489 -0
  136. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  137. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  138. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  139. data/rbi/generated/prism/dsl.rbi +751 -0
  140. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  141. data/rbi/generated/prism/lex_compat.rbi +162 -0
  142. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  143. data/rbi/generated/prism/node.rbi +15143 -0
  144. data/rbi/generated/prism/node_ext.rbi +259 -0
  145. data/rbi/generated/prism/node_find.rbi +79 -0
  146. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  147. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  148. data/rbi/generated/prism/parse_result.rbi +776 -0
  149. data/rbi/generated/prism/pattern.rbi +134 -0
  150. data/rbi/generated/prism/reflection.rbi +89 -0
  151. data/rbi/generated/prism/relocation.rbi +329 -0
  152. data/rbi/generated/prism/serialize.rbi +185 -0
  153. data/rbi/generated/prism/string_query.rbi +43 -0
  154. data/rbi/generated/prism/translation.rbi +7 -0
  155. data/rbi/generated/prism/visitor.rbi +650 -0
  156. data/rbi/generated/prism.rbi +86 -0
  157. data/rbi/prism/translation/parser_versions.rbi +5 -0
  158. data/rbi/rubyvm/node_find.rbi +14 -0
  159. data/sig/generated/prism/compiler.rbs +492 -0
  160. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  161. data/sig/generated/prism/dispatcher.rbs +1292 -0
  162. data/sig/generated/prism/dot_visitor.rbs +635 -0
  163. data/sig/generated/prism/dsl.rbs +928 -0
  164. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  165. data/sig/generated/prism/lex_compat.rbs +166 -0
  166. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  167. data/sig/generated/prism/node.rbs +21632 -0
  168. data/sig/generated/prism/node_ext.rbs +290 -0
  169. data/sig/generated/prism/node_find.rbs +90 -0
  170. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  171. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  172. data/sig/generated/prism/parse_result.rbs +871 -0
  173. data/sig/generated/prism/pattern.rbs +156 -0
  174. data/sig/generated/prism/reflection.rbs +101 -0
  175. data/sig/generated/prism/relocation.rbs +428 -0
  176. data/sig/generated/prism/serialize.rbs +200 -0
  177. data/sig/generated/prism/string_query.rbs +43 -0
  178. data/sig/generated/prism/translation.rbs +8 -0
  179. data/sig/generated/prism/visitor.rbs +1111 -0
  180. data/sig/generated/prism.rbs +84 -0
  181. data/sig/manifest.yaml +2 -0
  182. data/src/arena.c +122 -0
  183. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  184. data/src/{util/pm_char.c → char.c} +28 -57
  185. data/src/comments.c +455 -0
  186. data/src/constant_pool.c +406 -0
  187. data/src/diagnostic.c +120 -68
  188. data/src/encoding.c +8 -3
  189. data/src/errors_format.c +519 -0
  190. data/src/{util/pm_integer.c → integer.c} +38 -22
  191. data/src/json.c +5733 -0
  192. data/src/line_offset_list.c +100 -0
  193. data/src/list.c +24 -0
  194. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  195. data/src/node.c +3465 -7787
  196. data/src/options.c +161 -54
  197. data/src/parser.c +371 -0
  198. data/src/prettyprint.c +421 -354
  199. data/src/prism.c +8503 -7280
  200. data/src/regexp.c +960 -45
  201. data/src/serialize.c +2434 -2048
  202. data/src/source.c +524 -0
  203. data/src/static_literals.c +303 -360
  204. data/src/string_query.c +180 -0
  205. data/src/stringy.c +61 -0
  206. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  207. data/src/strpbrk.c +447 -0
  208. data/src/{token_type.c → tokens.c} +39 -18
  209. data/src/xxh3.c +292 -0
  210. metadata +141 -63
  211. data/ext/prism/api_pack.c +0 -276
  212. data/include/prism/defines.h +0 -260
  213. data/include/prism/pack.h +0 -163
  214. data/include/prism/regexp.h +0 -43
  215. data/include/prism/static_literals.h +0 -121
  216. data/include/prism/util/pm_buffer.h +0 -236
  217. data/include/prism/util/pm_char.h +0 -204
  218. data/include/prism/util/pm_constant_pool.h +0 -218
  219. data/include/prism/util/pm_integer.h +0 -130
  220. data/include/prism/util/pm_memchr.h +0 -29
  221. data/include/prism/util/pm_newline_list.h +0 -113
  222. data/include/prism/util/pm_string.h +0 -200
  223. data/lib/prism/pack.rb +0 -230
  224. data/lib/prism/parse_result/comments.rb +0 -188
  225. data/rbi/prism/compiler.rbi +0 -12
  226. data/rbi/prism/dsl.rbi +0 -524
  227. data/rbi/prism/inspect_visitor.rbi +0 -12
  228. data/rbi/prism/node.rbi +0 -8750
  229. data/rbi/prism/node_ext.rbi +0 -107
  230. data/rbi/prism/parse_result.rbi +0 -404
  231. data/rbi/prism/reflection.rbi +0 -58
  232. data/rbi/prism/string_query.rbi +0 -12
  233. data/rbi/prism/visitor.rbi +0 -473
  234. data/rbi/prism.rbi +0 -63
  235. data/sig/prism/compiler.rbs +0 -9
  236. data/sig/prism/dispatcher.rbs +0 -19
  237. data/sig/prism/dot_visitor.rbs +0 -6
  238. data/sig/prism/dsl.rbs +0 -351
  239. data/sig/prism/inspect_visitor.rbs +0 -22
  240. data/sig/prism/lex_compat.rbs +0 -10
  241. data/sig/prism/node.rbs +0 -4042
  242. data/sig/prism/node_ext.rbs +0 -149
  243. data/sig/prism/pack.rbs +0 -43
  244. data/sig/prism/parse_result/comments.rbs +0 -38
  245. data/sig/prism/parse_result.rbs +0 -197
  246. data/sig/prism/pattern.rbs +0 -13
  247. data/sig/prism/reflection.rbs +0 -50
  248. data/sig/prism/relocation.rbs +0 -185
  249. data/sig/prism/serialize.rbs +0 -8
  250. data/sig/prism/string_query.rbs +0 -11
  251. data/sig/prism/visitor.rbs +0 -169
  252. data/sig/prism.rbs +0 -272
  253. data/src/pack.c +0 -509
  254. data/src/util/pm_constant_pool.c +0 -342
  255. data/src/util/pm_list.c +0 -49
  256. data/src/util/pm_newline_list.c +0 -125
  257. data/src/util/pm_string.c +0 -381
  258. data/src/util/pm_strpbrk.c +0 -206
data/src/util/pm_string.c DELETED
@@ -1,381 +0,0 @@
1
- #include "prism/util/pm_string.h"
2
-
3
- static const uint8_t empty_source[] = "";
4
-
5
- /**
6
- * Returns the size of the pm_string_t struct. This is necessary to allocate the
7
- * correct amount of memory in the FFI backend.
8
- */
9
- PRISM_EXPORTED_FUNCTION size_t
10
- pm_string_sizeof(void) {
11
- return sizeof(pm_string_t);
12
- }
13
-
14
- /**
15
- * Initialize a shared string that is based on initial input.
16
- */
17
- void
18
- pm_string_shared_init(pm_string_t *string, const uint8_t *start, const uint8_t *end) {
19
- assert(start <= end);
20
-
21
- *string = (pm_string_t) {
22
- .type = PM_STRING_SHARED,
23
- .source = start,
24
- .length = (size_t) (end - start)
25
- };
26
- }
27
-
28
- /**
29
- * Initialize an owned string that is responsible for freeing allocated memory.
30
- */
31
- void
32
- pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length) {
33
- *string = (pm_string_t) {
34
- .type = PM_STRING_OWNED,
35
- .source = source,
36
- .length = length
37
- };
38
- }
39
-
40
- /**
41
- * Initialize a constant string that doesn't own its memory source.
42
- */
43
- void
44
- pm_string_constant_init(pm_string_t *string, const char *source, size_t length) {
45
- *string = (pm_string_t) {
46
- .type = PM_STRING_CONSTANT,
47
- .source = (const uint8_t *) source,
48
- .length = length
49
- };
50
- }
51
-
52
- #ifdef _WIN32
53
- /**
54
- * Represents a file handle on Windows, where the path will need to be freed
55
- * when the file is closed.
56
- */
57
- typedef struct {
58
- /** The path to the file, which will become allocated memory. */
59
- WCHAR *path;
60
-
61
- /** The handle to the file, which will start as uninitialized memory. */
62
- HANDLE file;
63
- } pm_string_file_handle_t;
64
-
65
- /**
66
- * Open the file indicated by the filepath parameter for reading on Windows.
67
- * Perform any kind of normalization that needs to happen on the filepath.
68
- */
69
- static pm_string_init_result_t
70
- pm_string_file_handle_open(pm_string_file_handle_t *handle, const char *filepath) {
71
- int length = MultiByteToWideChar(CP_UTF8, 0, filepath, -1, NULL, 0);
72
- if (length == 0) return PM_STRING_INIT_ERROR_GENERIC;
73
-
74
- handle->path = xmalloc(sizeof(WCHAR) * ((size_t) length));
75
- if ((handle->path == NULL) || (MultiByteToWideChar(CP_UTF8, 0, filepath, -1, handle->path, length) == 0)) {
76
- xfree(handle->path);
77
- return PM_STRING_INIT_ERROR_GENERIC;
78
- }
79
-
80
- handle->file = CreateFileW(handle->path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, NULL);
81
- if (handle->file == INVALID_HANDLE_VALUE) {
82
- pm_string_init_result_t result = PM_STRING_INIT_ERROR_GENERIC;
83
-
84
- if (GetLastError() == ERROR_ACCESS_DENIED) {
85
- DWORD attributes = GetFileAttributesW(handle->path);
86
- if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {
87
- result = PM_STRING_INIT_ERROR_DIRECTORY;
88
- }
89
- }
90
-
91
- xfree(handle->path);
92
- return result;
93
- }
94
-
95
- return PM_STRING_INIT_SUCCESS;
96
- }
97
-
98
- /**
99
- * Close the file handle and free the path.
100
- */
101
- static void
102
- pm_string_file_handle_close(pm_string_file_handle_t *handle) {
103
- xfree(handle->path);
104
- CloseHandle(handle->file);
105
- }
106
- #endif
107
-
108
- /**
109
- * Read the file indicated by the filepath parameter into source and load its
110
- * contents and size into the given `pm_string_t`. The given `pm_string_t`
111
- * should be freed using `pm_string_free` when it is no longer used.
112
- *
113
- * We want to use demand paging as much as possible in order to avoid having to
114
- * read the entire file into memory (which could be detrimental to performance
115
- * for large files). This means that if we're on windows we'll use
116
- * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
117
- * `mmap`, and on other POSIX systems we'll use `read`.
118
- */
119
- PRISM_EXPORTED_FUNCTION pm_string_init_result_t
120
- pm_string_mapped_init(pm_string_t *string, const char *filepath) {
121
- #ifdef _WIN32
122
- // Open the file for reading.
123
- pm_string_file_handle_t handle;
124
- pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
125
- if (result != PM_STRING_INIT_SUCCESS) return result;
126
-
127
- // Get the file size.
128
- DWORD file_size = GetFileSize(handle.file, NULL);
129
- if (file_size == INVALID_FILE_SIZE) {
130
- pm_string_file_handle_close(&handle);
131
- return PM_STRING_INIT_ERROR_GENERIC;
132
- }
133
-
134
- // If the file is empty, then we don't need to do anything else, we'll set
135
- // the source to a constant empty string and return.
136
- if (file_size == 0) {
137
- pm_string_file_handle_close(&handle);
138
- *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = empty_source, .length = 0 };
139
- return PM_STRING_INIT_SUCCESS;
140
- }
141
-
142
- // Create a mapping of the file.
143
- HANDLE mapping = CreateFileMapping(handle.file, NULL, PAGE_READONLY, 0, 0, NULL);
144
- if (mapping == NULL) {
145
- pm_string_file_handle_close(&handle);
146
- return PM_STRING_INIT_ERROR_GENERIC;
147
- }
148
-
149
- // Map the file into memory.
150
- uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0);
151
- CloseHandle(mapping);
152
- pm_string_file_handle_close(&handle);
153
-
154
- if (source == NULL) {
155
- return PM_STRING_INIT_ERROR_GENERIC;
156
- }
157
-
158
- *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
159
- return PM_STRING_INIT_SUCCESS;
160
- #elif defined(_POSIX_MAPPED_FILES)
161
- // Open the file for reading
162
- int fd = open(filepath, O_RDONLY);
163
- if (fd == -1) {
164
- return PM_STRING_INIT_ERROR_GENERIC;
165
- }
166
-
167
- // Stat the file to get the file size
168
- struct stat sb;
169
- if (fstat(fd, &sb) == -1) {
170
- close(fd);
171
- return PM_STRING_INIT_ERROR_GENERIC;
172
- }
173
-
174
- // Ensure it is a file and not a directory
175
- if (S_ISDIR(sb.st_mode)) {
176
- close(fd);
177
- return PM_STRING_INIT_ERROR_DIRECTORY;
178
- }
179
-
180
- // mmap the file descriptor to virtually get the contents
181
- size_t size = (size_t) sb.st_size;
182
- uint8_t *source = NULL;
183
-
184
- if (size == 0) {
185
- close(fd);
186
- *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = empty_source, .length = 0 };
187
- return PM_STRING_INIT_SUCCESS;
188
- }
189
-
190
- source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
191
- if (source == MAP_FAILED) {
192
- close(fd);
193
- return PM_STRING_INIT_ERROR_GENERIC;
194
- }
195
-
196
- close(fd);
197
- *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
198
- return PM_STRING_INIT_SUCCESS;
199
- #else
200
- return pm_string_file_init(string, filepath);
201
- #endif
202
- }
203
-
204
- /**
205
- * Read the file indicated by the filepath parameter into source and load its
206
- * contents and size into the given `pm_string_t`. The given `pm_string_t`
207
- * should be freed using `pm_string_free` when it is no longer used.
208
- */
209
- PRISM_EXPORTED_FUNCTION pm_string_init_result_t
210
- pm_string_file_init(pm_string_t *string, const char *filepath) {
211
- #ifdef _WIN32
212
- // Open the file for reading.
213
- pm_string_file_handle_t handle;
214
- pm_string_init_result_t result = pm_string_file_handle_open(&handle, filepath);
215
- if (result != PM_STRING_INIT_SUCCESS) return result;
216
-
217
- // Get the file size.
218
- DWORD file_size = GetFileSize(handle.file, NULL);
219
- if (file_size == INVALID_FILE_SIZE) {
220
- pm_string_file_handle_close(&handle);
221
- return PM_STRING_INIT_ERROR_GENERIC;
222
- }
223
-
224
- // If the file is empty, then we don't need to do anything else, we'll set
225
- // the source to a constant empty string and return.
226
- if (file_size == 0) {
227
- pm_string_file_handle_close(&handle);
228
- *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = empty_source, .length = 0 };
229
- return PM_STRING_INIT_SUCCESS;
230
- }
231
-
232
- // Create a buffer to read the file into.
233
- uint8_t *source = xmalloc(file_size);
234
- if (source == NULL) {
235
- pm_string_file_handle_close(&handle);
236
- return PM_STRING_INIT_ERROR_GENERIC;
237
- }
238
-
239
- // Read the contents of the file
240
- DWORD bytes_read;
241
- if (!ReadFile(handle.file, source, file_size, &bytes_read, NULL)) {
242
- pm_string_file_handle_close(&handle);
243
- return PM_STRING_INIT_ERROR_GENERIC;
244
- }
245
-
246
- // Check the number of bytes read
247
- if (bytes_read != file_size) {
248
- xfree(source);
249
- pm_string_file_handle_close(&handle);
250
- return PM_STRING_INIT_ERROR_GENERIC;
251
- }
252
-
253
- pm_string_file_handle_close(&handle);
254
- *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = (size_t) file_size };
255
- return PM_STRING_INIT_SUCCESS;
256
- #elif defined(PRISM_HAS_FILESYSTEM)
257
- // Open the file for reading
258
- int fd = open(filepath, O_RDONLY);
259
- if (fd == -1) {
260
- return PM_STRING_INIT_ERROR_GENERIC;
261
- }
262
-
263
- // Stat the file to get the file size
264
- struct stat sb;
265
- if (fstat(fd, &sb) == -1) {
266
- close(fd);
267
- return PM_STRING_INIT_ERROR_GENERIC;
268
- }
269
-
270
- // Ensure it is a file and not a directory
271
- if (S_ISDIR(sb.st_mode)) {
272
- close(fd);
273
- return PM_STRING_INIT_ERROR_DIRECTORY;
274
- }
275
-
276
- // Check the size to see if it's empty
277
- size_t size = (size_t) sb.st_size;
278
- if (size == 0) {
279
- close(fd);
280
- *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = empty_source, .length = 0 };
281
- return PM_STRING_INIT_SUCCESS;
282
- }
283
-
284
- size_t length = (size_t) size;
285
- uint8_t *source = xmalloc(length);
286
- if (source == NULL) {
287
- close(fd);
288
- return PM_STRING_INIT_ERROR_GENERIC;
289
- }
290
-
291
- long bytes_read = (long) read(fd, source, length);
292
- close(fd);
293
-
294
- if (bytes_read == -1) {
295
- xfree(source);
296
- return PM_STRING_INIT_ERROR_GENERIC;
297
- }
298
-
299
- *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = length };
300
- return PM_STRING_INIT_SUCCESS;
301
- #else
302
- (void) string;
303
- (void) filepath;
304
- perror("pm_string_file_init is not implemented for this platform");
305
- return PM_STRING_INIT_ERROR_GENERIC;
306
- #endif
307
- }
308
-
309
- /**
310
- * Ensure the string is owned. If it is not, then reinitialize it as owned and
311
- * copy over the previous source.
312
- */
313
- void
314
- pm_string_ensure_owned(pm_string_t *string) {
315
- if (string->type == PM_STRING_OWNED) return;
316
-
317
- size_t length = pm_string_length(string);
318
- const uint8_t *source = pm_string_source(string);
319
-
320
- uint8_t *memory = xmalloc(length);
321
- if (!memory) return;
322
-
323
- pm_string_owned_init(string, memory, length);
324
- memcpy((void *) string->source, source, length);
325
- }
326
-
327
- /**
328
- * Compare the underlying lengths and bytes of two strings. Returns 0 if the
329
- * strings are equal, a negative number if the left string is less than the
330
- * right string, and a positive number if the left string is greater than the
331
- * right string.
332
- */
333
- int
334
- pm_string_compare(const pm_string_t *left, const pm_string_t *right) {
335
- size_t left_length = pm_string_length(left);
336
- size_t right_length = pm_string_length(right);
337
-
338
- if (left_length < right_length) {
339
- return -1;
340
- } else if (left_length > right_length) {
341
- return 1;
342
- }
343
-
344
- return memcmp(pm_string_source(left), pm_string_source(right), left_length);
345
- }
346
-
347
- /**
348
- * Returns the length associated with the string.
349
- */
350
- PRISM_EXPORTED_FUNCTION size_t
351
- pm_string_length(const pm_string_t *string) {
352
- return string->length;
353
- }
354
-
355
- /**
356
- * Returns the start pointer associated with the string.
357
- */
358
- PRISM_EXPORTED_FUNCTION const uint8_t *
359
- pm_string_source(const pm_string_t *string) {
360
- return string->source;
361
- }
362
-
363
- /**
364
- * Free the associated memory of the given string.
365
- */
366
- PRISM_EXPORTED_FUNCTION void
367
- pm_string_free(pm_string_t *string) {
368
- void *memory = (void *) string->source;
369
-
370
- if (string->type == PM_STRING_OWNED) {
371
- xfree(memory);
372
- #ifdef PRISM_HAS_MMAP
373
- } else if (string->type == PM_STRING_MAPPED && string->length) {
374
- #if defined(_WIN32)
375
- UnmapViewOfFile(memory);
376
- #elif defined(_POSIX_MAPPED_FILES)
377
- munmap(memory, string->length);
378
- #endif
379
- #endif /* PRISM_HAS_MMAP */
380
- }
381
- }
@@ -1,206 +0,0 @@
1
- #include "prism/util/pm_strpbrk.h"
2
-
3
- /**
4
- * Add an invalid multibyte character error to the parser.
5
- */
6
- static inline void
7
- pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) {
8
- pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
9
- }
10
-
11
- /**
12
- * Set the explicit encoding for the parser to the current encoding.
13
- */
14
- static inline void
15
- pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
16
- if (parser->explicit_encoding != NULL) {
17
- if (parser->explicit_encoding == parser->encoding) {
18
- // Okay, we already locked to this encoding.
19
- } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
20
- // Not okay, we already found a Unicode escape sequence and this
21
- // conflicts.
22
- pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
23
- } else {
24
- // Should not be anything else.
25
- assert(false && "unreachable");
26
- }
27
- }
28
-
29
- parser->explicit_encoding = parser->encoding;
30
- }
31
-
32
- /**
33
- * This is the default path.
34
- */
35
- static inline const uint8_t *
36
- pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
37
- size_t index = 0;
38
-
39
- while (index < maximum) {
40
- if (strchr((const char *) charset, source[index]) != NULL) {
41
- return source + index;
42
- }
43
-
44
- if (source[index] < 0x80) {
45
- index++;
46
- } else {
47
- size_t width = pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index));
48
-
49
- if (width > 0) {
50
- index += width;
51
- } else if (!validate) {
52
- index++;
53
- } else {
54
- // At this point we know we have an invalid multibyte character.
55
- // We'll walk forward as far as we can until we find the next
56
- // valid character so that we don't spam the user with a ton of
57
- // the same kind of error.
58
- const size_t start = index;
59
-
60
- do {
61
- index++;
62
- } while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
63
-
64
- pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
65
- }
66
- }
67
- }
68
-
69
- return NULL;
70
- }
71
-
72
- /**
73
- * This is the path when the encoding is ASCII-8BIT.
74
- */
75
- static inline const uint8_t *
76
- pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
77
- size_t index = 0;
78
-
79
- while (index < maximum) {
80
- if (strchr((const char *) charset, source[index]) != NULL) {
81
- return source + index;
82
- }
83
-
84
- if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
85
- index++;
86
- }
87
-
88
- return NULL;
89
- }
90
-
91
- /**
92
- * This is the slow path that does care about the encoding.
93
- */
94
- static inline const uint8_t *
95
- pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
96
- size_t index = 0;
97
- const pm_encoding_t *encoding = parser->encoding;
98
-
99
- while (index < maximum) {
100
- if (strchr((const char *) charset, source[index]) != NULL) {
101
- return source + index;
102
- }
103
-
104
- if (source[index] < 0x80) {
105
- index++;
106
- } else {
107
- size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
108
- if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
109
-
110
- if (width > 0) {
111
- index += width;
112
- } else if (!validate) {
113
- index++;
114
- } else {
115
- // At this point we know we have an invalid multibyte character.
116
- // We'll walk forward as far as we can until we find the next
117
- // valid character so that we don't spam the user with a ton of
118
- // the same kind of error.
119
- const size_t start = index;
120
-
121
- do {
122
- index++;
123
- } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
124
-
125
- pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
126
- }
127
- }
128
- }
129
-
130
- return NULL;
131
- }
132
-
133
- /**
134
- * This is the fast path that does not care about the encoding because we know
135
- * the encoding only supports single-byte characters.
136
- */
137
- static inline const uint8_t *
138
- pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
139
- size_t index = 0;
140
- const pm_encoding_t *encoding = parser->encoding;
141
-
142
- while (index < maximum) {
143
- if (strchr((const char *) charset, source[index]) != NULL) {
144
- return source + index;
145
- }
146
-
147
- if (source[index] < 0x80 || !validate) {
148
- index++;
149
- } else {
150
- size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
151
- pm_strpbrk_explicit_encoding_set(parser, source, width);
152
-
153
- if (width > 0) {
154
- index += width;
155
- } else {
156
- // At this point we know we have an invalid multibyte character.
157
- // We'll walk forward as far as we can until we find the next
158
- // valid character so that we don't spam the user with a ton of
159
- // the same kind of error.
160
- const size_t start = index;
161
-
162
- do {
163
- index++;
164
- } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
165
-
166
- pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
167
- }
168
- }
169
- }
170
-
171
- return NULL;
172
- }
173
-
174
- /**
175
- * Here we have rolled our own version of strpbrk. The standard library strpbrk
176
- * has undefined behavior when the source string is not null-terminated. We want
177
- * to support strings that are not null-terminated because pm_parse does not
178
- * have the contract that the string is null-terminated. (This is desirable
179
- * because it means the extension can call pm_parse with the result of a call to
180
- * mmap).
181
- *
182
- * The standard library strpbrk also does not support passing a maximum length
183
- * to search. We want to support this for the reason mentioned above, but we
184
- * also don't want it to stop on null bytes. Ruby actually allows null bytes
185
- * within strings, comments, regular expressions, etc. So we need to be able to
186
- * skip past them.
187
- *
188
- * Finally, we want to support encodings wherein the charset could contain
189
- * characters that are trailing bytes of multi-byte characters. For example, in
190
- * Shift_JIS, the backslash character can be a trailing byte. In that case we
191
- * need to take a slower path and iterate one multi-byte character at a time.
192
- */
193
- const uint8_t *
194
- pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate) {
195
- if (length <= 0) {
196
- return NULL;
197
- } else if (!parser->encoding_changed) {
198
- return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
199
- } else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
200
- return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
201
- } else if (parser->encoding->multibyte) {
202
- return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
203
- } else {
204
- return pm_strpbrk_single_byte(parser, source, charset, (size_t) length, validate);
205
- }
206
- }