prism 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +51 -42
- data/Makefile +8 -8
- data/README.md +3 -1
- data/docs/fuzzing.md +5 -5
- data/ext/prism/extension.h +1 -1
- data/include/prism/internal/comments.h +2 -1
- data/include/prism/internal/parser.h +26 -15
- data/include/prism/internal/static_literals.h +24 -64
- data/include/prism/version.h +2 -2
- data/lib/prism/serialize.rb +1 -1
- data/prism.gemspec +1 -1
- data/src/arena.c +6 -1
- data/src/comments.c +9 -8
- data/src/diagnostic.c +16 -4
- data/src/integer.c +6 -1
- data/src/prism.c +51 -50
- data/src/static_literals.c +280 -405
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8e2f8907c10234cbde0a461b0674bc64139201a2d846bc2aa55e7ff416e3bac8
|
|
4
|
+
data.tar.gz: 396fd09bb9efc62dc076783701bb98fc9576b722ff9e2b4e89366eeef02f606e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 844a68c1e7f2ebed766ec0a2b9f68ce2849e0205fd8339a04310a5b8725cdb820c1c3d04b077c11f112dff974cd74a1d29a9c7dd1c8f5d3a61efc3cfc1dbc50c
|
|
7
|
+
data.tar.gz: 90ac3995ffabefe8943d3159ac6f6b5a215b214c7d74a3c42c4fbf9abd3a3bfb12af32cee27747b28b1c5467ff0edf956d1346caa065fc81a29b7c3daa24699f
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [2.1.0] - 2026-10-28
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Add `Version::CRuby4_0` to the Rust bindings. `Version::CRuby3_5` remains available but is deprecated in favor of the new name; both select Ruby 4.0 syntax.
|
|
14
|
+
- Let embedders set the allocation size for the first block allocated in an arena via `PM_ARENA_INITIAL_SIZE`.
|
|
15
|
+
- Let embedders set the minimum initial size for the constant pool via `PM_CONSTANT_POOL_MIN_CAPACITY`.
|
|
16
|
+
|
|
9
17
|
## [2.0.0] - 2026-10-21
|
|
10
18
|
|
|
11
19
|
### Highlights
|
|
@@ -837,45 +845,46 @@ Note that for this major version, almost every change listed below is breaking.
|
|
|
837
845
|
|
|
838
846
|
- 🎉 Initial release! 🎉
|
|
839
847
|
|
|
840
|
-
[unreleased]: https://github.com/ruby/
|
|
841
|
-
[2.
|
|
842
|
-
[
|
|
843
|
-
[1.
|
|
844
|
-
[1.8.
|
|
845
|
-
[1.
|
|
846
|
-
[1.
|
|
847
|
-
[1.
|
|
848
|
-
[1.5.
|
|
849
|
-
[1.5.
|
|
850
|
-
[1.
|
|
851
|
-
[1.
|
|
852
|
-
[1.
|
|
853
|
-
[1.
|
|
854
|
-
[1.
|
|
855
|
-
[0.
|
|
856
|
-
[0.
|
|
857
|
-
[0.
|
|
858
|
-
[0.
|
|
859
|
-
[0.
|
|
860
|
-
[0.
|
|
861
|
-
[0.
|
|
862
|
-
[0.
|
|
863
|
-
[0.
|
|
864
|
-
[0.
|
|
865
|
-
[0.
|
|
866
|
-
[0.
|
|
867
|
-
[0.
|
|
868
|
-
[0.
|
|
869
|
-
[0.17.
|
|
870
|
-
[0.
|
|
871
|
-
[0.
|
|
872
|
-
[0.15.
|
|
873
|
-
[0.
|
|
874
|
-
[0.
|
|
875
|
-
[0.
|
|
876
|
-
[0.
|
|
877
|
-
[0.
|
|
878
|
-
[0.
|
|
879
|
-
[0.
|
|
880
|
-
[0.
|
|
881
|
-
[0.
|
|
848
|
+
[unreleased]: https://github.com/ruby/prism3/compare/v2.1.0...HEAD
|
|
849
|
+
[2.1.0]: https://github.com/ruby/prism3/compare/v2.0.0...v2.1.0
|
|
850
|
+
[2.0.0]: https://github.com/ruby/prism3/compare/v1.9.0...v2.0.0
|
|
851
|
+
[1.9.0]: https://github.com/ruby/prism3/compare/v1.8.1...v1.9.0
|
|
852
|
+
[1.8.1]: https://github.com/ruby/prism3/compare/v1.8.0...v1.8.1
|
|
853
|
+
[1.8.0]: https://github.com/ruby/prism3/compare/v1.7.0...v1.8.0
|
|
854
|
+
[1.7.0]: https://github.com/ruby/prism3/compare/v1.6.0...v1.7.0
|
|
855
|
+
[1.6.0]: https://github.com/ruby/prism3/compare/v1.5.2...v1.6.0
|
|
856
|
+
[1.5.2]: https://github.com/ruby/prism3/compare/v1.5.1...v1.5.2
|
|
857
|
+
[1.5.1]: https://github.com/ruby/prism3/compare/v1.5.0...v1.5.1
|
|
858
|
+
[1.5.0]: https://github.com/ruby/prism3/compare/v1.4.0...v1.5.0
|
|
859
|
+
[1.4.0]: https://github.com/ruby/prism3/compare/v1.3.0...v1.4.0
|
|
860
|
+
[1.3.0]: https://github.com/ruby/prism3/compare/v1.2.0...v1.3.0
|
|
861
|
+
[1.2.0]: https://github.com/ruby/prism3/compare/v1.1.0...v1.2.0
|
|
862
|
+
[1.1.0]: https://github.com/ruby/prism3/compare/v1.0.0...v1.1.0
|
|
863
|
+
[1.0.0]: https://github.com/ruby/prism3/compare/v0.30.0...v1.0.0
|
|
864
|
+
[0.30.0]: https://github.com/ruby/prism3/compare/v0.29.0...v0.30.0
|
|
865
|
+
[0.29.0]: https://github.com/ruby/prism3/compare/v0.28.0...v0.29.0
|
|
866
|
+
[0.28.0]: https://github.com/ruby/prism3/compare/v0.27.0...v0.28.0
|
|
867
|
+
[0.27.0]: https://github.com/ruby/prism3/compare/v0.26.0...v0.27.0
|
|
868
|
+
[0.26.0]: https://github.com/ruby/prism3/compare/v0.25.0...v0.26.0
|
|
869
|
+
[0.25.0]: https://github.com/ruby/prism3/compare/v0.24.0...v0.25.0
|
|
870
|
+
[0.24.0]: https://github.com/ruby/prism3/compare/v0.23.0...v0.24.0
|
|
871
|
+
[0.23.0]: https://github.com/ruby/prism3/compare/v0.22.0...v0.23.0
|
|
872
|
+
[0.22.0]: https://github.com/ruby/prism3/compare/v0.21.0...v0.22.0
|
|
873
|
+
[0.21.0]: https://github.com/ruby/prism3/compare/v0.20.0...v0.21.0
|
|
874
|
+
[0.20.0]: https://github.com/ruby/prism3/compare/v0.19.0...v0.20.0
|
|
875
|
+
[0.19.0]: https://github.com/ruby/prism3/compare/v0.18.0...v0.19.0
|
|
876
|
+
[0.18.0]: https://github.com/ruby/prism3/compare/v0.17.1...v0.18.0
|
|
877
|
+
[0.17.1]: https://github.com/ruby/prism3/compare/v0.17.0...v0.17.1
|
|
878
|
+
[0.17.0]: https://github.com/ruby/prism3/compare/v0.16.0...v0.17.0
|
|
879
|
+
[0.16.0]: https://github.com/ruby/prism3/compare/v0.15.1...v0.16.0
|
|
880
|
+
[0.15.1]: https://github.com/ruby/prism3/compare/v0.15.0...v0.15.1
|
|
881
|
+
[0.15.0]: https://github.com/ruby/prism3/compare/v0.14.0...v0.15.0
|
|
882
|
+
[0.14.0]: https://github.com/ruby/prism3/compare/v0.13.0...v0.14.0
|
|
883
|
+
[0.13.0]: https://github.com/ruby/prism3/compare/v0.12.0...v0.13.0
|
|
884
|
+
[0.12.0]: https://github.com/ruby/prism3/compare/v0.11.0...v0.12.0
|
|
885
|
+
[0.11.0]: https://github.com/ruby/prism3/compare/v0.10.0...v0.11.0
|
|
886
|
+
[0.10.0]: https://github.com/ruby/prism3/compare/v0.9.0...v0.10.0
|
|
887
|
+
[0.9.0]: https://github.com/ruby/prism3/compare/v0.8.0...v0.9.0
|
|
888
|
+
[0.8.0]: https://github.com/ruby/prism3/compare/v0.7.0...v0.8.0
|
|
889
|
+
[0.7.0]: https://github.com/ruby/prism3/compare/v0.6.0...v0.7.0
|
|
890
|
+
[0.6.0]: https://github.com/ruby/prism3/compare/d60531...v0.6.0
|
data/Makefile
CHANGED
|
@@ -6,7 +6,7 @@ Q1 = $(V:1=)
|
|
|
6
6
|
Q = $(Q1:0=@)
|
|
7
7
|
ECHO1 = $(V:1=@ :)
|
|
8
8
|
ECHO = $(ECHO1:0=@ echo)
|
|
9
|
-
FUZZ_OUTPUT_DIR = $(CURDIR)/fuzz/output
|
|
9
|
+
FUZZ_OUTPUT_DIR = $(CURDIR)/test/fuzz/output
|
|
10
10
|
|
|
11
11
|
SOEXT ?= $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
|
|
12
12
|
|
|
@@ -94,7 +94,7 @@ build/static/%.o: src/%.c Makefile $(HEADERS)
|
|
|
94
94
|
$(Q) $(MAKEDIRS) $(@D)
|
|
95
95
|
$(Q) $(CC) $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
|
|
96
96
|
|
|
97
|
-
build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
|
|
97
|
+
build/fuzz.%: $(SOURCES) test/fuzz/%.c test/fuzz/fuzz.c
|
|
98
98
|
$(ECHO) "building $* fuzzer"
|
|
99
99
|
$(Q) $(MAKEDIRS) $(@D)
|
|
100
100
|
$(ECHO) "building main fuzz binary"
|
|
@@ -102,16 +102,16 @@ build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
|
|
|
102
102
|
$(ECHO) "building cmplog binary"
|
|
103
103
|
$(Q) AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
|
|
104
104
|
|
|
105
|
-
build/fuzz.heisenbug.%: $(SOURCES) fuzz/%.c fuzz/heisenbug.c
|
|
105
|
+
build/fuzz.heisenbug.%: $(SOURCES) test/fuzz/%.c test/fuzz/heisenbug.c
|
|
106
106
|
$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
|
|
107
107
|
|
|
108
108
|
fuzz-debug:
|
|
109
109
|
$(ECHO) "entering debug shell"
|
|
110
|
-
$(Q) docker run -it --rm -e HISTFILE=/prism/fuzz/output/.bash_history -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz
|
|
110
|
+
$(Q) docker run -it --rm -e HISTFILE=/prism/test/fuzz/output/.bash_history -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz
|
|
111
111
|
|
|
112
|
-
fuzz-docker-build: fuzz/docker/Dockerfile
|
|
112
|
+
fuzz-docker-build: test/fuzz/docker/Dockerfile
|
|
113
113
|
$(ECHO) "building docker image"
|
|
114
|
-
$(Q) docker build -t prism/fuzz fuzz/docker/
|
|
114
|
+
$(Q) docker build -t prism/fuzz test/fuzz/docker/
|
|
115
115
|
|
|
116
116
|
fuzz-run-%: FORCE fuzz-docker-build
|
|
117
117
|
$(ECHO) "generating templates"
|
|
@@ -120,11 +120,11 @@ fuzz-run-%: FORCE fuzz-docker-build
|
|
|
120
120
|
$(Q) docker run --rm -v $(CURDIR):/prism prism/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
|
|
121
121
|
$(ECHO) "starting AFL++ run"
|
|
122
122
|
$(Q) $(MAKEDIRS) $(FUZZ_OUTPUT_DIR)/$*
|
|
123
|
-
$(Q) docker run -it --rm -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
|
|
123
|
+
$(Q) docker run -it --rm -v $(CURDIR):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz /bin/bash -c "./test/fuzz/$*.sh /fuzz_output/$*"
|
|
124
124
|
FORCE:
|
|
125
125
|
|
|
126
126
|
fuzz-clean:
|
|
127
|
-
$(Q) $(RMALL) fuzz/output
|
|
127
|
+
$(Q) $(RMALL) test/fuzz/output
|
|
128
128
|
|
|
129
129
|
clean:
|
|
130
130
|
$(Q) $(RMALL) build
|
data/README.md
CHANGED
|
@@ -49,7 +49,6 @@ The repository contains the infrastructure for both a shared library (libprism)
|
|
|
49
49
|
│ └── prism
|
|
50
50
|
│ ├── extconf.rb configuration to generate the Makefile for the native extension
|
|
51
51
|
│ └── extension.c the native extension that interacts with libprism
|
|
52
|
-
├── fuzz files related to fuzz testing
|
|
53
52
|
├── gemfiles gemfiles used by different Ruby versions in CI
|
|
54
53
|
├── include
|
|
55
54
|
│ ├── prism header files for the shared library
|
|
@@ -75,6 +74,9 @@ The repository contains the infrastructure for both a shared library (libprism)
|
|
|
75
74
|
├── templates contains ERB templates generated by templates/template.rb
|
|
76
75
|
│ └── template.rb generates code from the nodes and tokens configured by config.yml
|
|
77
76
|
└── test
|
|
77
|
+
├── cpp tests for the C++ bindings
|
|
78
|
+
├── embedded smoke test for bare-metal targets
|
|
79
|
+
├── fuzz files related to fuzz testing
|
|
78
80
|
└── prism
|
|
79
81
|
├── fixtures Ruby code used for testing
|
|
80
82
|
└── snapshots snapshots of generated syntax trees corresponding to fixtures
|
data/docs/fuzzing.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# Fuzzing
|
|
2
2
|
|
|
3
|
-
We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `fuzz` directory, which has the following structure:
|
|
3
|
+
We use fuzzing to test the various entrypoints to the library. The fuzzer we use is [AFL++](https://aflplus.plus). All files related to fuzzing live within the `test/fuzz` directory, which has the following structure:
|
|
4
4
|
|
|
5
5
|
```
|
|
6
|
-
fuzz
|
|
6
|
+
test/fuzz
|
|
7
7
|
├── corpus
|
|
8
8
|
│ └── parse fuzzing corpus for parsing (a symlink to our fixtures)
|
|
9
9
|
├── dict a AFL++ dictionary containing various tokens
|
|
@@ -57,12 +57,12 @@ make build/fuzz.heisenbug.parse
|
|
|
57
57
|
Triaging crashes and hangs is easier when the inputs are as short as possible. In the fuzz container, an entire crash or hang directory can be minimized using
|
|
58
58
|
|
|
59
59
|
```
|
|
60
|
-
./fuzz/tools/minimize.sh <directory>
|
|
60
|
+
./test/fuzz/tools/minimize.sh <directory>
|
|
61
61
|
```
|
|
62
62
|
|
|
63
63
|
e.g.
|
|
64
64
|
```
|
|
65
|
-
./fuzz/tools/minimize.sh fuzz/output/parse/default/crashes
|
|
65
|
+
./test/fuzz/tools/minimize.sh test/fuzz/output/parse/default/crashes
|
|
66
66
|
```
|
|
67
67
|
|
|
68
68
|
This may take a long time. In the crash/hang directory, for each input file there will appear a minimized version with the extension `.min` appended.
|
|
@@ -70,7 +70,7 @@ This may take a long time. In the crash/hang directory, for each input file ther
|
|
|
70
70
|
Backtraces for crashes (not hangs) can be generated en masse with
|
|
71
71
|
|
|
72
72
|
```
|
|
73
|
-
./fuzz/tools/backtrace.sh <directory>
|
|
73
|
+
./test/fuzz/tools/backtrace.sh <directory>
|
|
74
74
|
```
|
|
75
75
|
|
|
76
76
|
Files with basename equal to the input file name with extension `.bt` will be created e.g.
|
data/ext/prism/extension.h
CHANGED
|
@@ -46,7 +46,8 @@ typedef struct {
|
|
|
46
46
|
/*
|
|
47
47
|
* Sort the parser's comment list in place by start offset, then assign every
|
|
48
48
|
* comment to the node that owns it. This is called after parsing when the
|
|
49
|
-
* attach_comments option is set
|
|
49
|
+
* attach_comments option is set, before the parser's scratch arena is released,
|
|
50
|
+
* because the walk allocates from it.
|
|
50
51
|
*/
|
|
51
52
|
void pm_comments_attach(pm_parser_t *parser, const pm_node_t *root);
|
|
52
53
|
|
|
@@ -272,13 +272,6 @@ typedef struct pm_lex_mode {
|
|
|
272
272
|
struct pm_lex_mode *prev;
|
|
273
273
|
} pm_lex_mode_t;
|
|
274
274
|
|
|
275
|
-
/*
|
|
276
|
-
* We pre-allocate a certain number of lex states in order to avoid having to
|
|
277
|
-
* call malloc too many times while parsing. You really shouldn't need more than
|
|
278
|
-
* this because you only really nest deeply when doing string interpolation.
|
|
279
|
-
*/
|
|
280
|
-
#define PM_LEX_STACK_SIZE 4
|
|
281
|
-
|
|
282
275
|
/*
|
|
283
276
|
* While parsing, we keep track of a stack of contexts. This is helpful for
|
|
284
277
|
* error recovery so that we can pop back to a previous context when we hit a
|
|
@@ -596,9 +589,20 @@ struct pm_parser_t {
|
|
|
596
589
|
/* The arena used for all AST-lifetime allocations. Caller-owned. */
|
|
597
590
|
pm_arena_t *arena;
|
|
598
591
|
|
|
599
|
-
/*
|
|
592
|
+
/*
|
|
593
|
+
* The arena used for parser metadata (comments, diagnostics, etc.), which
|
|
594
|
+
* lives as long as the parser.
|
|
595
|
+
*/
|
|
600
596
|
pm_arena_t metadata_arena;
|
|
601
597
|
|
|
598
|
+
/*
|
|
599
|
+
* The arena used for state that is only needed while parsing, such as
|
|
600
|
+
* scopes, locals, and contexts. It is released when pm_parse returns, so
|
|
601
|
+
* nothing allocated in it can be reachable from the tree or from the
|
|
602
|
+
* parser's metadata.
|
|
603
|
+
*/
|
|
604
|
+
pm_arena_t scratch_arena;
|
|
605
|
+
|
|
602
606
|
/*
|
|
603
607
|
* The next node identifier that will be assigned. This is a unique
|
|
604
608
|
* identifier used to track nodes such that the syntax tree can be dropped
|
|
@@ -648,16 +652,23 @@ struct pm_parser_t {
|
|
|
648
652
|
*/
|
|
649
653
|
pm_state_stack_t accepts_block_stack;
|
|
650
654
|
|
|
651
|
-
/* A stack of lex modes. */
|
|
655
|
+
/* A stack of lex modes, linked through their prev pointers. */
|
|
652
656
|
struct {
|
|
653
657
|
/* The current mode of the lexer. */
|
|
654
658
|
pm_lex_mode_t *current;
|
|
655
659
|
|
|
656
|
-
/*
|
|
657
|
-
|
|
660
|
+
/*
|
|
661
|
+
* The mode at the bottom of the stack, which is never popped. Every
|
|
662
|
+
* other mode is allocated from the scratch arena.
|
|
663
|
+
*/
|
|
664
|
+
pm_lex_mode_t base;
|
|
658
665
|
|
|
659
|
-
/*
|
|
660
|
-
|
|
666
|
+
/*
|
|
667
|
+
* Lex modes that have been popped, kept for reuse by the next push.
|
|
668
|
+
* They are allocated from the scratch arena, so they are never freed
|
|
669
|
+
* individually.
|
|
670
|
+
*/
|
|
671
|
+
pm_lex_mode_t *free;
|
|
661
672
|
} lex_modes;
|
|
662
673
|
|
|
663
674
|
/* The pointer to the start of the source. */
|
|
@@ -717,7 +728,7 @@ struct pm_parser_t {
|
|
|
717
728
|
|
|
718
729
|
/*
|
|
719
730
|
* Scopes that have been popped, kept for reuse by the next push. Scopes
|
|
720
|
-
* are allocated from the
|
|
731
|
+
* are allocated from the scratch arena, so they are never freed
|
|
721
732
|
* individually.
|
|
722
733
|
*/
|
|
723
734
|
pm_scope_t *free_scopes;
|
|
@@ -727,7 +738,7 @@ struct pm_parser_t {
|
|
|
727
738
|
|
|
728
739
|
/*
|
|
729
740
|
* Context nodes that have been popped, kept for reuse by the next push.
|
|
730
|
-
* Context nodes are allocated from the
|
|
741
|
+
* Context nodes are allocated from the scratch arena, so they are never
|
|
731
742
|
* freed individually.
|
|
732
743
|
*/
|
|
733
744
|
pm_context_node_t *free_contexts;
|
|
@@ -1,91 +1,51 @@
|
|
|
1
1
|
#ifndef PRISM_INTERNAL_STATIC_LITERALS_H
|
|
2
2
|
#define PRISM_INTERNAL_STATIC_LITERALS_H
|
|
3
3
|
|
|
4
|
-
#include "prism/internal/encoding.h"
|
|
5
|
-
|
|
6
4
|
#include "prism/ast.h"
|
|
7
5
|
#include "prism/buffer.h"
|
|
8
|
-
#include "prism/
|
|
9
|
-
|
|
10
|
-
/*
|
|
11
|
-
* An internal hash table for a set of nodes.
|
|
12
|
-
*/
|
|
13
|
-
typedef struct {
|
|
14
|
-
/* The array of nodes in the hash table. */
|
|
15
|
-
pm_node_t **nodes;
|
|
16
|
-
|
|
17
|
-
/* The size of the hash table. */
|
|
18
|
-
uint32_t size;
|
|
19
|
-
|
|
20
|
-
/* The space that has been allocated in the hash table. */
|
|
21
|
-
uint32_t capacity;
|
|
22
|
-
} pm_node_hash_t;
|
|
6
|
+
#include "prism/parser.h"
|
|
23
7
|
|
|
24
8
|
/*
|
|
25
9
|
* Certain sets of nodes (hash keys and when clauses) check for duplicate nodes
|
|
26
10
|
* to alert the user of potential issues. To do this, we keep a set of the nodes
|
|
27
11
|
* that have been seen so far, and compare whenever we find a new node.
|
|
28
|
-
*
|
|
29
|
-
* We bucket the nodes based on their type to minimize the number of comparisons
|
|
30
|
-
* that need to be performed.
|
|
31
12
|
*/
|
|
32
13
|
typedef struct {
|
|
33
|
-
/*
|
|
34
|
-
|
|
35
|
-
*/
|
|
36
|
-
pm_node_hash_t integer_nodes;
|
|
14
|
+
/* The node stored in this slot, or NULL if the slot is empty. */
|
|
15
|
+
pm_node_t *node;
|
|
37
16
|
|
|
38
|
-
/*
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
pm_node_hash_t float_nodes;
|
|
17
|
+
/* The hash of the node, kept so that it is never recomputed. */
|
|
18
|
+
uint64_t hash;
|
|
19
|
+
} pm_static_literals_entry_t;
|
|
42
20
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
* This is the set of StringNode and SourceFileNode instances.
|
|
50
|
-
*/
|
|
51
|
-
pm_node_hash_t string_nodes;
|
|
52
|
-
|
|
53
|
-
/*
|
|
54
|
-
* This is the set of RegularExpressionNode instances.
|
|
55
|
-
*/
|
|
56
|
-
pm_node_hash_t regexp_nodes;
|
|
21
|
+
/*
|
|
22
|
+
* The number of nodes a set holds directly before it moves them into a hash
|
|
23
|
+
* table. Most sets are this small, so they are searched linearly without
|
|
24
|
+
* hashing their nodes or allocating any memory.
|
|
25
|
+
*/
|
|
26
|
+
#define PM_STATIC_LITERALS_INLINE_CAPACITY 4
|
|
57
27
|
|
|
28
|
+
typedef struct {
|
|
58
29
|
/*
|
|
59
|
-
*
|
|
30
|
+
* The nodes in the set while it has no hash table. Once the set outgrows
|
|
31
|
+
* this array, its nodes move into the hash table and this is unused.
|
|
60
32
|
*/
|
|
61
|
-
|
|
33
|
+
pm_node_t *nodes[PM_STATIC_LITERALS_INLINE_CAPACITY];
|
|
62
34
|
|
|
63
|
-
/*
|
|
64
|
-
|
|
65
|
-
*/
|
|
66
|
-
pm_node_t *true_node;
|
|
35
|
+
/* The array of entries in the hash table, or NULL if there is none. */
|
|
36
|
+
pm_static_literals_entry_t *entries;
|
|
67
37
|
|
|
68
|
-
/*
|
|
69
|
-
|
|
70
|
-
*/
|
|
71
|
-
pm_node_t *false_node;
|
|
72
|
-
|
|
73
|
-
/*
|
|
74
|
-
* A pointer to the last NilNode instance that was inserted, or NULL.
|
|
75
|
-
*/
|
|
76
|
-
pm_node_t *nil_node;
|
|
38
|
+
/* The number of nodes in the set. */
|
|
39
|
+
uint32_t size;
|
|
77
40
|
|
|
78
|
-
/*
|
|
79
|
-
|
|
80
|
-
* NULL.
|
|
81
|
-
*/
|
|
82
|
-
pm_node_t *source_encoding_node;
|
|
41
|
+
/* The number of entries allocated in the hash table, or 0 if there is none. */
|
|
42
|
+
uint32_t capacity;
|
|
83
43
|
} pm_static_literals_t;
|
|
84
44
|
|
|
85
45
|
/*
|
|
86
46
|
* Add a node to the set of static literals.
|
|
87
47
|
*/
|
|
88
|
-
pm_node_t * pm_static_literals_add(const
|
|
48
|
+
pm_node_t * pm_static_literals_add(const pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node, bool replace);
|
|
89
49
|
|
|
90
50
|
/*
|
|
91
51
|
* Free the internal memory associated with the given static literals set.
|
|
@@ -95,6 +55,6 @@ void pm_static_literals_free(pm_static_literals_t *literals);
|
|
|
95
55
|
/*
|
|
96
56
|
* Create a string-based representation of the given static literal.
|
|
97
57
|
*/
|
|
98
|
-
void pm_static_literal_inspect(pm_buffer_t *buffer, const
|
|
58
|
+
void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
|
|
99
59
|
|
|
100
60
|
#endif
|
data/include/prism/version.h
CHANGED
|
@@ -20,7 +20,7 @@ extern "C" {
|
|
|
20
20
|
/**
|
|
21
21
|
* The minor version of the Prism library as an int.
|
|
22
22
|
*/
|
|
23
|
-
#define PRISM_VERSION_MINOR
|
|
23
|
+
#define PRISM_VERSION_MINOR 1
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* The patch version of the Prism library as an int.
|
|
@@ -30,7 +30,7 @@ extern "C" {
|
|
|
30
30
|
/**
|
|
31
31
|
* The version of the Prism library as a constant string.
|
|
32
32
|
*/
|
|
33
|
-
#define PRISM_VERSION "2.
|
|
33
|
+
#define PRISM_VERSION "2.1.0"
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* The prism version and the serialization format.
|
data/lib/prism/serialize.rb
CHANGED
data/prism.gemspec
CHANGED
data/src/arena.c
CHANGED
|
@@ -12,8 +12,13 @@
|
|
|
12
12
|
*/
|
|
13
13
|
#define PM_ARENA_BLOCK_SIZE(data_size) (offsetof(pm_arena_block_t, data) + (data_size))
|
|
14
14
|
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Initial block data size: 8 KB. An embedder on a small heap can define a
|
|
17
|
+
* smaller one; a parse of a short program then takes blocks of that size.
|
|
18
|
+
*/
|
|
19
|
+
#ifndef PM_ARENA_INITIAL_SIZE
|
|
16
20
|
#define PM_ARENA_INITIAL_SIZE 8192
|
|
21
|
+
#endif
|
|
17
22
|
|
|
18
23
|
/** Double the block size every this many blocks. */
|
|
19
24
|
#define PM_ARENA_GROWTH_INTERVAL 8
|
data/src/comments.c
CHANGED
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
|
|
12
12
|
/* The state used while walking the tree to find the owner of each comment. */
|
|
13
13
|
typedef struct {
|
|
14
|
-
/*
|
|
14
|
+
/*
|
|
15
|
+
* The arena for memory that is only needed during the walk, which is the
|
|
16
|
+
* scratch arena of the parser.
|
|
17
|
+
*/
|
|
15
18
|
pm_arena_t *arena;
|
|
16
19
|
|
|
17
20
|
/* The start of the source that was parsed. */
|
|
@@ -389,15 +392,15 @@ pm_comments_attach(pm_parser_t *parser, const pm_node_t *root) {
|
|
|
389
392
|
uint32_t size = (uint32_t) parser->comment_list.size;
|
|
390
393
|
if (size == 0) return;
|
|
391
394
|
|
|
392
|
-
pm_arena_t arena =
|
|
395
|
+
pm_arena_t *arena = &parser->scratch_arena;
|
|
393
396
|
pm_comments_walk_t walk = {
|
|
394
|
-
.arena =
|
|
397
|
+
.arena = arena,
|
|
395
398
|
.source = parser->start,
|
|
396
399
|
.comments = pm_comments_sort(parser, size),
|
|
397
400
|
.size = size,
|
|
398
|
-
.starts = (uint32_t *) pm_arena_alloc(
|
|
399
|
-
.attached = (const pm_node_t **) pm_arena_zalloc(
|
|
400
|
-
.unclaimed = (uint32_t *) pm_arena_alloc(
|
|
401
|
+
.starts = (uint32_t *) pm_arena_alloc(arena, sizeof(uint32_t) * size, PRISM_ALIGNOF(uint32_t)),
|
|
402
|
+
.attached = (const pm_node_t **) pm_arena_zalloc(arena, sizeof(pm_node_t *) * size, PRISM_ALIGNOF(pm_node_t *)),
|
|
403
|
+
.unclaimed = (uint32_t *) pm_arena_alloc(arena, sizeof(uint32_t) * (size + 1), PRISM_ALIGNOF(uint32_t))
|
|
401
404
|
};
|
|
402
405
|
|
|
403
406
|
for (uint32_t index = 0; index < size; index++) walk.starts[index] = walk.comments[index]->location.start;
|
|
@@ -443,8 +446,6 @@ pm_comments_attach(pm_parser_t *parser, const pm_node_t *root) {
|
|
|
443
446
|
attachments[range->index + range->length++] = index;
|
|
444
447
|
}
|
|
445
448
|
|
|
446
|
-
pm_arena_cleanup(&arena);
|
|
447
|
-
|
|
448
449
|
parser->comment_attachments = (pm_comment_attachments_t) {
|
|
449
450
|
.comments = walk.comments,
|
|
450
451
|
.attachments = attachments,
|
data/src/diagnostic.c
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
#include <stdarg.h>
|
|
21
21
|
#include <stdio.h>
|
|
22
22
|
#include <stdlib.h>
|
|
23
|
+
#include <string.h>
|
|
23
24
|
|
|
24
25
|
#define PM_DIAGNOSTIC_ID_MAX 327
|
|
25
26
|
|
|
@@ -863,11 +864,16 @@ pm_diagnostic_list_append(pm_arena_t *arena, pm_list_t *list, uint32_t start, ui
|
|
|
863
864
|
*/
|
|
864
865
|
void
|
|
865
866
|
pm_diagnostic_list_append_format(pm_arena_t *arena, pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...) {
|
|
867
|
+
/* Format the message into a buffer on the stack, which nearly every message
|
|
868
|
+
* fits in. The result is the length of the whole message whether or not it
|
|
869
|
+
* fit, so a single pass both formats the message and sizes it. */
|
|
870
|
+
char stack[256];
|
|
871
|
+
|
|
866
872
|
va_list arguments;
|
|
867
873
|
va_start(arguments, diag_id);
|
|
868
874
|
|
|
869
875
|
const char *format = pm_diagnostic_id_message(diag_id);
|
|
870
|
-
int result = vsnprintf(
|
|
876
|
+
int result = vsnprintf(stack, sizeof(stack), format, arguments);
|
|
871
877
|
va_end(arguments);
|
|
872
878
|
|
|
873
879
|
if (result < 0) {
|
|
@@ -879,9 +885,15 @@ pm_diagnostic_list_append_format(pm_arena_t *arena, pm_list_t *list, uint32_t st
|
|
|
879
885
|
size_t message_length = (size_t) (result + 1);
|
|
880
886
|
char *message = (char *) pm_arena_alloc(arena, message_length, 1);
|
|
881
887
|
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
888
|
+
if (message_length <= sizeof(stack)) {
|
|
889
|
+
memcpy(message, stack, message_length);
|
|
890
|
+
} else {
|
|
891
|
+
/* The message was cut off on the stack, so format it again into an
|
|
892
|
+
* allocation of the right size. */
|
|
893
|
+
va_start(arguments, diag_id);
|
|
894
|
+
vsnprintf(message, message_length, format, arguments);
|
|
895
|
+
va_end(arguments);
|
|
896
|
+
}
|
|
885
897
|
|
|
886
898
|
*diagnostic = (pm_diagnostic_t) {
|
|
887
899
|
.location = { .start = start, .length = length },
|
data/src/integer.c
CHANGED
|
@@ -3,8 +3,13 @@
|
|
|
3
3
|
#include "prism/internal/allocator.h"
|
|
4
4
|
#include "prism/internal/buffer.h"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
/* Include sys/types.h before inttypes.h to work around an issue with certain
|
|
7
|
+
* versions of GCC and newlib which causes omission of PRIu64 and the other
|
|
8
|
+
* 64-bit macros. */
|
|
9
|
+
#include <sys/types.h>
|
|
7
10
|
#include <inttypes.h>
|
|
11
|
+
|
|
12
|
+
#include <assert.h>
|
|
8
13
|
#include <stdbool.h>
|
|
9
14
|
#include <stddef.h>
|
|
10
15
|
#include <stdlib.h>
|