rouge-raku 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +13 -0
- data/Gemfile +12 -0
- data/LICENSE +21 -0
- data/README.md +122 -0
- data/lib/rouge/demos/raku +6 -0
- data/lib/rouge/lexers/raku.rb +1001 -0
- data/lib/rouge/raku/version.rb +7 -0
- data/lib/rouge/raku.rb +37 -0
- data/lib/rouge-raku.rb +3 -0
- data/rouge-raku.gemspec +27 -0
- metadata +78 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 2abd02a3dd1ca15acdd031b967aa8b6276d8e10661664f9657806895788e7980
|
|
4
|
+
data.tar.gz: 9e18210ce8e22ed1c47629aab3c92f0c6793a03c95cf52929268fb347efae27d
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 70bb9a66d622e2fde9f7eeabeba64c0133b1c2b25556a8442549f5f23781b586db555cd3be03fb3e52766864e5542e45cca997497643969d1dba8d2b1ee47549
|
|
7
|
+
data.tar.gz: a92cf7b41df0f44c904af968299717d643224987fc11a2ccc1c8a2d53e297494b411fb0c43dc0b412089352c5104f629b7f394bf6e6e35d52dd38e1f2244b4f6
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- A lexer for Raku: comments, Pod, variables, numbers, keywords, types,
|
|
8
|
+
operators, strings and quote-like forms, heredocs, regexes, substitutions
|
|
9
|
+
and grammars.
|
|
10
|
+
- Registers the tag `raku` and the file extensions `.raku`, `.rakumod`,
|
|
11
|
+
`.rakutest` and `.rakudoc`, and recognises Raku source by a `raku` or
|
|
12
|
+
`rakudo` shebang or a leading `use v6`.
|
|
13
|
+
- Works with Rouge 3.26 and later, up to and including Rouge 5.
|
data/Gemfile
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source 'https://rubygems.org'
|
|
4
|
+
|
|
5
|
+
gemspec
|
|
6
|
+
|
|
7
|
+
# to run the specs against a version of Rouge other than the latest
|
|
8
|
+
rouge_version = ENV['ROUGE_VERSION'].to_s
|
|
9
|
+
gem 'rouge', rouge_version unless rouge_version.empty?
|
|
10
|
+
|
|
11
|
+
gem 'minitest', '~> 5.0'
|
|
12
|
+
gem 'rake', '~> 13.0'
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Naoum Hankache
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# rouge-raku
|
|
2
|
+
|
|
3
|
+
A [Raku](https://raku.org) lexer for the [Rouge](https://rouge.jneen.ca/)
|
|
4
|
+
syntax highlighter, packaged as a plugin gem. Rouge does not highlight Raku
|
|
5
|
+
on its own; with this gem loaded, it does.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
Add it to your `Gemfile`:
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
gem 'rouge-raku'
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
or install it yourself:
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
gem install rouge-raku
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
The gem has to be loaded for Rouge to know about Raku. How to do that
|
|
24
|
+
depends on what is running Rouge.
|
|
25
|
+
|
|
26
|
+
### Ruby
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
require 'rouge'
|
|
30
|
+
require 'rouge-raku'
|
|
31
|
+
|
|
32
|
+
Rouge.highlight('say "Hello, World!";', 'raku', 'html')
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
### Command line
|
|
36
|
+
|
|
37
|
+
```sh
|
|
38
|
+
rougify highlight -r rouge-raku hello.raku
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
### Jekyll
|
|
42
|
+
|
|
43
|
+
Put the gem in the `jekyll_plugins` group of your `Gemfile`:
|
|
44
|
+
|
|
45
|
+
```ruby
|
|
46
|
+
group :jekyll_plugins do
|
|
47
|
+
gem 'rouge-raku'
|
|
48
|
+
end
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or list it under `plugins` in `_config.yml`:
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
plugins:
|
|
55
|
+
- rouge-raku
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Then use `raku` as the language of a code block:
|
|
59
|
+
|
|
60
|
+
````markdown
|
|
61
|
+
```raku
|
|
62
|
+
say "Hello, World!";
|
|
63
|
+
```
|
|
64
|
+
````
|
|
65
|
+
|
|
66
|
+
On GitHub Pages, the built-in build only loads GitHub's own list of plugins
|
|
67
|
+
and will not load this gem. Build the site with a GitHub Actions workflow
|
|
68
|
+
instead.
|
|
69
|
+
|
|
70
|
+
### Asciidoctor
|
|
71
|
+
|
|
72
|
+
Set Rouge as the highlighter in the document:
|
|
73
|
+
|
|
74
|
+
```asciidoc
|
|
75
|
+
:source-highlighter: rouge
|
|
76
|
+
|
|
77
|
+
[source,raku]
|
|
78
|
+
----
|
|
79
|
+
say "Hello, World!";
|
|
80
|
+
----
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
and load the gem when converting it:
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
asciidoctor -r rouge-raku document.adoc
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## What it recognises
|
|
90
|
+
|
|
91
|
+
The lexer has the tag `raku`. Rouge picks it by itself for files ending in
|
|
92
|
+
`.raku`, `.rakumod`, `.rakutest` or `.rakudoc`, and for source that starts
|
|
93
|
+
with a `raku` or `rakudo` shebang or with `use v6`.
|
|
94
|
+
|
|
95
|
+
## Compatibility
|
|
96
|
+
|
|
97
|
+
Ruby 3.0 or later, and Rouge 3.26 or later, up to and including Rouge 5.
|
|
98
|
+
|
|
99
|
+
## Development
|
|
100
|
+
|
|
101
|
+
```sh
|
|
102
|
+
bundle install
|
|
103
|
+
bundle exec rake
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
To run the specs against another version of Rouge:
|
|
107
|
+
|
|
108
|
+
```sh
|
|
109
|
+
ROUGE_VERSION=4.7.0 bundle install
|
|
110
|
+
ROUGE_VERSION=4.7.0 bundle exec rake
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The files in `spec/snippets/raku` each hold a piece of Raku and the tokens
|
|
114
|
+
it should produce, and the spec checks the lexer against them.
|
|
115
|
+
|
|
116
|
+
The layout follows Rouge's own conventions (`lib/rouge/lexers/raku.rb`,
|
|
117
|
+
`lib/rouge/demos/raku`, `spec/lexers/raku_spec.rb`,
|
|
118
|
+
`spec/visual/samples/raku`) so the lexer can be offered upstream later.
|
|
119
|
+
|
|
120
|
+
## License
|
|
121
|
+
|
|
122
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,1001 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*- #
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Rouge
|
|
5
|
+
module Lexers
|
|
6
|
+
class Raku < RegexLexer
|
|
7
|
+
title "Raku"
|
|
8
|
+
desc "The Raku programming language (raku.org)"
|
|
9
|
+
|
|
10
|
+
tag 'raku'
|
|
11
|
+
|
|
12
|
+
filenames '*.raku', '*.rakumod', '*.rakutest', '*.rakudoc'
|
|
13
|
+
mimetypes 'text/x-raku', 'application/x-raku'
|
|
14
|
+
|
|
15
|
+
def self.detect?(text)
|
|
16
|
+
return true if text.shebang?(/raku|rakudo/)
|
|
17
|
+
return true if text =~ /\A(?:\s*(?:#.*)?\n)*\s*use\s+v6\b/
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# words that declare something: scope, routine and package declarators
|
|
21
|
+
DECLARATORS = %w(
|
|
22
|
+
anon augment class constant enum grammar has knowhow macro
|
|
23
|
+
method module multi my only our package proto regex role rule
|
|
24
|
+
state sub submethod subset supersede token unit
|
|
25
|
+
).freeze
|
|
26
|
+
|
|
27
|
+
NAMESPACE_KEYWORDS = %w(
|
|
28
|
+
import need no require use
|
|
29
|
+
).freeze
|
|
30
|
+
|
|
31
|
+
# phasers, control flow, traits and other reserved words
|
|
32
|
+
KEYWORDS = %w(
|
|
33
|
+
BEGIN CATCH CHECK CLOSE CONTROL DOC END ENTER FIRST INIT KEEP
|
|
34
|
+
LAST LEAVE NEXT POST PRE QUIT UNDO also default do does eager
|
|
35
|
+
else elsif export for gather given handles hides hyper if is
|
|
36
|
+
last lazy let loop made make native next of once orwith
|
|
37
|
+
proceed quietly race react redo repeat repr required return
|
|
38
|
+
return-rw returns rw sink start succeed supply symbol temp
|
|
39
|
+
trusts try unless until when whenever where while will with
|
|
40
|
+
without
|
|
41
|
+
).freeze
|
|
42
|
+
|
|
43
|
+
# names of the traits that can follow "is"
|
|
44
|
+
TRAITS = %w(
|
|
45
|
+
DEPRECATED assoc built cached copy default dynamic equiv
|
|
46
|
+
export hidden-from-USAGE hidden-from-backtrace
|
|
47
|
+
implementation-detail looser native nodal pure raw readonly
|
|
48
|
+
repr required rw symbol test-assertion tighter
|
|
49
|
+
).freeze
|
|
50
|
+
|
|
51
|
+
# Routines that can be called as a plain word: say "hi", map(...). These
|
|
52
|
+
# are builtins wherever they appear. Routines of the Test module are
|
|
53
|
+
# included, as a lexer cannot know whether it has been loaded.
|
|
54
|
+
BUILTINS = %w(
|
|
55
|
+
abs acos acosec acosech acosh acotan acotanh all any append
|
|
56
|
+
asec asech asin asinh atan atan2 atanh atomic-assign
|
|
57
|
+
atomic-dec-fetch atomic-fetch atomic-fetch-add
|
|
58
|
+
atomic-fetch-dec atomic-fetch-inc atomic-fetch-sub
|
|
59
|
+
atomic-inc-fetch await bag bail-out callframe callsame
|
|
60
|
+
callwith can-ok cas categorize ceiling cglobal chars chdir
|
|
61
|
+
chmod chomp chop chown chr chrs cis classify close cmp-ok comb
|
|
62
|
+
combinations copy cos cosec cosech cosh cotan cotanh cross dd
|
|
63
|
+
deepmap defined diag die dies-ok dir does-ok done done-testing
|
|
64
|
+
duckmap elems emit end EVAL eval-dies-ok eval-lives-ok
|
|
65
|
+
EVALFILE exit exits-ok exp explicitly-manage expmod fail
|
|
66
|
+
fails-like fc first flat flip floor flunk full-barrier get
|
|
67
|
+
getc gist grep hash head index indir is-approx is-deeply
|
|
68
|
+
is-prime isa-ok isnt item join keys kv lastcall lc like lines
|
|
69
|
+
link list lives-ok log log10 log2 lsb MAIN map max min minmax
|
|
70
|
+
mix mkdir move msb nano nativecast nativesizeof nextcallee
|
|
71
|
+
nextsame nextwith nodemap nok none not note now ok one open
|
|
72
|
+
ord ords pack pairs parse-base parse-names pass periods
|
|
73
|
+
permutations pick plan pop prepend print printf produce prompt
|
|
74
|
+
push put rand reduce rename repeated repl report reverse
|
|
75
|
+
rindex rmdir roll roots rotate round roundrobin run samecase
|
|
76
|
+
samemark samewith say sec sech set shell shift sign signal sin
|
|
77
|
+
sinh skip skip-rest sleep sleep-timer sleep-until slip slurp
|
|
78
|
+
snap snapper snip snitch so sort splice split sprintf spurt
|
|
79
|
+
sqrt squish srand subbuf-rw substr substr-rw subtest sum
|
|
80
|
+
symlink tail take take-rw tan tanh tc tclc throws-like time
|
|
81
|
+
todo trim trim-leading trim-trailing truncate uc undefine
|
|
82
|
+
unimatch uniname uninames uniparse uniprop uniprops unique
|
|
83
|
+
unival unlike unlink unpack unpolar unshift USAGE use-ok val
|
|
84
|
+
values warn wordcase words zip
|
|
85
|
+
).freeze
|
|
86
|
+
|
|
87
|
+
# Routines that only exist as methods. "name" is the builtin in
|
|
88
|
+
# "$x.name", but on its own it is a word the user chose ("name => 1",
|
|
89
|
+
# "sub name"), so these are only builtins after a dot.
|
|
90
|
+
METHODS = %w(
|
|
91
|
+
abs2rel absolute accept ACCEPTS accessed acquire act action
|
|
92
|
+
actions add add_attribute add_enum_value add_fallback
|
|
93
|
+
add_method add_parent add_private_method add_role add_trustee
|
|
94
|
+
adverb after allocate allof alternative-names annotations
|
|
95
|
+
antipair antipairs anyof app_lifetime arch archname are args
|
|
96
|
+
arity ASSIGN-KEY ASSIGN-POS assuming ast at AT-KEY AT-POS
|
|
97
|
+
attributes auth backtrace base base-repeating basename batch
|
|
98
|
+
before BIND-KEY BIND-POS bind-stderr bind-stdin bind-stdout
|
|
99
|
+
bind-udp bits bless block bool-only bounds break Bridge broken
|
|
100
|
+
BUILD build-date bytecode-size bytes cache CALL-ME
|
|
101
|
+
calling-package can cancel candidates cando canonpath caps
|
|
102
|
+
caption capture catdir categorize-list catfile catpath cause
|
|
103
|
+
changed child child-name child-typename chunks classify-list
|
|
104
|
+
cleanup clone close-stdin closed code codes collate column
|
|
105
|
+
command comment compiler compose compose_type composer concise
|
|
106
|
+
condition config configure_destroy configure_type_checking
|
|
107
|
+
conj connect constraints construct contains contents count
|
|
108
|
+
count-only cpu-cores cpu-usage CREATE create_type created cue
|
|
109
|
+
curdir curupdir d day day-fraction day-of-month day-of-week
|
|
110
|
+
day-of-year daycount days-in-month days-in-year dd-mm-yyyy
|
|
111
|
+
declaration decode decoder default DEFINITE delayed DELETE-KEY
|
|
112
|
+
DELETE-POS denominator desc DESTROY destroyers dev devnull
|
|
113
|
+
devtype did-you-mean dir-sep dir-with-entries dirname
|
|
114
|
+
DISTROnames do does dynamic e eager earlier enclosing encode
|
|
115
|
+
encoder encoding ends-with enum_from_value enum_value_list
|
|
116
|
+
enum_values enums eof err exception excludes-max excludes-min
|
|
117
|
+
EXISTS-KEY EXISTS-POS exitcode expected extension f feature
|
|
118
|
+
file filename find_method find_method_qualified finish
|
|
119
|
+
first-date-in-month flatmap flush fmt format formatter freeze
|
|
120
|
+
from from-list from-loop from-posix from-slurpy full get_value
|
|
121
|
+
got grab grabpairs handle handled handles hardware
|
|
122
|
+
has_accessor headers hh-mm-ss hidden hides hour HOW how hyper
|
|
123
|
+
id illegal im in in-range in-timezone indent indices infinite
|
|
124
|
+
infix inode install_method_cache instead int-bounds interval
|
|
125
|
+
invalid-str invert invocant is-absolute is-hidden
|
|
126
|
+
is-implementation-detail is-initial-thread is-int is-lazy
|
|
127
|
+
is-leap-year is-relative is-routine is-setting is-win
|
|
128
|
+
is_trusted is_type isa isNaN iterator julian-date keep kept
|
|
129
|
+
KERNELnames key keyof kill kxxv l lang last last-date-in-month
|
|
130
|
+
later lazy leading level line listen live local lock lookup
|
|
131
|
+
made make match maxpairs merge message method method_table
|
|
132
|
+
methods migrate minpairs minus minute misplaced mm-dd-yyyy
|
|
133
|
+
mode modified modified-julian-date modifier month mro multi
|
|
134
|
+
multi-invocant multiness my name named named_names narrow
|
|
135
|
+
native-descriptor new new-from-daycount new-from-pairs
|
|
136
|
+
new_type next next-handle next-interesting-index nice nl-in
|
|
137
|
+
nl-out norm nude numerator of offset offset-in-hours
|
|
138
|
+
offset-in-minutes old on-close on-switch opened operation
|
|
139
|
+
optional orig os-error osname out out-buffer outer
|
|
140
|
+
outer-caller-idx package package-kind package-name packages
|
|
141
|
+
pair pairup parameter params parent parent-name parents parse
|
|
142
|
+
parsefile parts path path-sep payload peer-host peer-port perl
|
|
143
|
+
phaser pickpairs pid placeholder plus polar poll polymod port
|
|
144
|
+
pos positional posix postfix postmatch precomp-ext
|
|
145
|
+
precomp-target pred prefix prematch print-nl print-to private
|
|
146
|
+
private_method_table proc protect pull-one push-all
|
|
147
|
+
push-at-least push-exactly push-until-lazy qualifier-type quit
|
|
148
|
+
r race radix raku range raw re read readchars readonly ready
|
|
149
|
+
reallocate reals reason rebless receive recv redispatcher redo
|
|
150
|
+
rel2abs relative release remove replace-with replacement REPR
|
|
151
|
+
reserved resolve restore result resume rethrow returns right
|
|
152
|
+
role roles_to_compose rolish rootdir rotor routine-type rw rwx
|
|
153
|
+
s schedule-on scheduler scope second seek send serial
|
|
154
|
+
set-instruments set_hidden set_name set_package set_rw
|
|
155
|
+
set_value setup_finalization shape share sibling sigil signals
|
|
156
|
+
signature sink sink-all skip-at-least skip-at-least-pull-one
|
|
157
|
+
skip-one slice slurp-rest slurpy socket-host socket-port
|
|
158
|
+
source source-package spawn SPEC splitdir splitpath stable
|
|
159
|
+
start started starts-with status stderr stdout sub_signature
|
|
160
|
+
subbuf subname subparse subst subst-mutate substr-eq succ
|
|
161
|
+
suffix summary t tap target target-name tell then throttle
|
|
162
|
+
throw timezone tmpdir to to-posix today toggle total trailing
|
|
163
|
+
trans tree truncated-to trusts try_acquire trying twigil type
|
|
164
|
+
type_captures typename udp uncaught_handler univals unlock
|
|
165
|
+
unset unwrap updir usage-name utc value VAR variable
|
|
166
|
+
verbose-config version VMnames volume vow w wait watch
|
|
167
|
+
watch-path week week-number week-year weekday-of-month WHAT
|
|
168
|
+
when WHERE WHEREFORE WHICH WHO whole-second WHY workaround
|
|
169
|
+
wrap write write-to x yada year yield yyyy-mm-dd z zip-latest
|
|
170
|
+
).freeze
|
|
171
|
+
|
|
172
|
+
BUILTIN_CLASSES = %w(
|
|
173
|
+
Any Array Associative AST atomicint Attribute Backtrace
|
|
174
|
+
Backtrace::Frame Bag Baggy BagHash Blob Block Bool Buf
|
|
175
|
+
Callable CallFrame Cancellation Capture CArray Channel Code
|
|
176
|
+
Complex ComplexStr Cool CurrentThreadScheduler Date Dateish
|
|
177
|
+
DateTime Distro Duration Encoding Exception Failure FatRat
|
|
178
|
+
Grammar Hash HyperWhatever Instant Int int16 int32 int64 int8
|
|
179
|
+
IntStr IO IO::ArgFiles IO::CatHandle IO::Handle
|
|
180
|
+
IO::Notification IO::Path IO::Path::Cygwin IO::Path::QNX
|
|
181
|
+
IO::Path::Unix IO::Path::Win32 IO::Pipe IO::Socket
|
|
182
|
+
IO::Socket::Async IO::Socket::INET IO::Spec IO::Spec::Cygwin
|
|
183
|
+
IO::Spec::QNX IO::Spec::Unix IO::Spec::Win32 IO::Special
|
|
184
|
+
Iterable Iterator Junction Kernel Label List Lock Lock::Async
|
|
185
|
+
long longlong Macro Map Match Metamodel::AttributeContainer
|
|
186
|
+
Metamodel::C3MRO Metamodel::ClassHOW Metamodel::EnumHOW
|
|
187
|
+
Metamodel::Finalization Metamodel::MethodContainer
|
|
188
|
+
Metamodel::MROBasedMethodDispatch
|
|
189
|
+
Metamodel::MultipleInheritance Metamodel::Naming
|
|
190
|
+
Metamodel::Primitives Metamodel::PrivateMethodContainer
|
|
191
|
+
Metamodel::RoleContainer Metamodel::Trusting Method Mix
|
|
192
|
+
MixHash Mixy Mu NFC NFD NFKC NFKD Num num32 num64 Numeric
|
|
193
|
+
NumStr ObjAt Order Pair Parameter Perl Pod::Block
|
|
194
|
+
Pod::Block::Code Pod::Block::Comment Pod::Block::Declarator
|
|
195
|
+
Pod::Block::Named Pod::Block::Para Pod::Block::Table
|
|
196
|
+
Pod::Heading Pod::Item Pointer Positional
|
|
197
|
+
PositionalBindFailover Proc Proc::Async Promise Proxy
|
|
198
|
+
PseudoStash QuantHash Range Rat Rational RatStr Real Regex
|
|
199
|
+
Routine Scalar Scheduler Semaphore Seq Set SetHash Setty
|
|
200
|
+
Signature size_t Slip Stash Str StrDistance Stringy Sub
|
|
201
|
+
Submethod Supplier Supplier::Preserving Supply Systemic Tap
|
|
202
|
+
Telemetry Telemetry::Instrument::Thread
|
|
203
|
+
Telemetry::Instrument::Usage Telemetry::Period
|
|
204
|
+
Telemetry::Sampler Thread ThreadPoolScheduler UInt uint16
|
|
205
|
+
uint32 uint64 uint8 Uni utf8 Variable Version VM Whatever
|
|
206
|
+
WhateverCode Allomorph Collation CompUnit
|
|
207
|
+
CompUnit::PrecompilationRepository CompUnit::Repository
|
|
208
|
+
CompUnit::Repository::FileSystem
|
|
209
|
+
CompUnit::Repository::Installation
|
|
210
|
+
CompUnit::Repository::Unknown Compiler Distribution
|
|
211
|
+
Distribution::Hash Distribution::Locally Distribution::Path
|
|
212
|
+
Distribution::Resource Encoding::Registry Endian Enumeration
|
|
213
|
+
ForeignCode Format Formatter HyperSeq IO::Notification::Change
|
|
214
|
+
IO::Path::Parts IO::Socket::Async::ListenSocket
|
|
215
|
+
IterationBuffer Lock::ConditionVariable
|
|
216
|
+
Metamodel::ConcreteRoleHOW Metamodel::CurriedRoleHOW
|
|
217
|
+
Metamodel::DefiniteHOW Metamodel::Documenting
|
|
218
|
+
Metamodel::MethodDelegation Metamodel::Mixins
|
|
219
|
+
Metamodel::PackageHOW Metamodel::ParametricRoleGroupHOW
|
|
220
|
+
Metamodel::ParametricRoleHOW Metamodel::RolePunning
|
|
221
|
+
Metamodel::Stashing Metamodel::TypePretense
|
|
222
|
+
Metamodel::Versioning Pod::Defn Pod::FormattingCode
|
|
223
|
+
PredictiveIterator PromiseStatus RaceSeq Raku RakuAST
|
|
224
|
+
Routine::WrapHandle Sequence Signal
|
|
225
|
+
Telemetry::Instrument::ThreadPool Unicode ValueObjAt array
|
|
226
|
+
blob8 blob16 blob32 blob64 buf8 buf16 buf32 buf64 byte int num
|
|
227
|
+
str uint
|
|
228
|
+
).freeze
|
|
229
|
+
|
|
230
|
+
WORD_OPERATORS = %w(
|
|
231
|
+
after and andthen before but cmp coll div eq eqv ff fff gcd ge
|
|
232
|
+
gt lcm le leg lt max min minmax mod ne not notandthen or
|
|
233
|
+
orelse R so unicmp x X xor xx Z
|
|
234
|
+
).freeze
|
|
235
|
+
|
|
236
|
+
CONSTANTS = %w(
|
|
237
|
+
π τ ∞ 𝑒 pi tau Inf NaN e i
|
|
238
|
+
).freeze
|
|
239
|
+
|
|
240
|
+
# symbolic operators, including the Unicode synonyms
|
|
241
|
+
SYMBOL_OPERATORS = %w(
|
|
242
|
+
... … <=> ==>> <<== ==> <== === =:= =~= !~~ ~~ ::= := ^..^ ..^
|
|
243
|
+
^.. .. ** ++ -- && || // ^^ ?? !! == != <= >= => %% +& +| +^
|
|
244
|
+
+< +> ~& ~| ~^ ~< ~> ?& ?| ?^ --> <-> -> ! ? + - * / % ~ | & ^
|
|
245
|
+
< > = . ∈ ∉ ∋ ∌ ⊂ ⊄ ⊃ ⊅ ⊆ ⊈ ⊇ ⊉ ≼ ≽ ∪ ∩ ∖ ⊖ ⊍ ⊎ ≤ ≥ ≠ ≅ × ÷ −
|
|
246
|
+
∘ ⚛ ≡ ≢ ⩶ ⩵ ∊ ∍
|
|
247
|
+
).freeze
|
|
248
|
+
|
|
249
|
+
# rules available in every grammar: <ws>, <alpha>, <before ...>, ...
|
|
250
|
+
REGEX_BUILTINS = %w(
|
|
251
|
+
alnum alpha after at before blank cntrl digit graph ident
|
|
252
|
+
lower print punct same space upper wb ws ww xdigit
|
|
253
|
+
).freeze
|
|
254
|
+
|
|
255
|
+
# Raku has a *lot* of possible bracketing characters. This list was
|
|
256
|
+
# lifted from STD.pm6 (https://github.com/perl6/std).
|
|
257
|
+
BRACKETS = %w(
|
|
258
|
+
() <> [] {} «» ༺༻ ༼༽ ᚛᚜ ‘’ ‚’ ‛’ “” „” ‟” ‹› ⁅⁆ ⁽⁾ ₍₎ ∈∋ ∉∌ ∊∍
|
|
259
|
+
∕⧵ ∼∽ ≃⋍ ≒≓ ≔≕ ≤≥ ≦≧ ≨≩ ≪≫ ≮≯ ≰≱ ≲≳ ≴≵ ≶≷ ≸≹ ≺≻ ≼≽ ≾≿ ⊀⊁ ⊂⊃ ⊄⊅
|
|
260
|
+
⊆⊇ ⊈⊉ ⊊⊋ ⊏⊐ ⊑⊒ ⊘⦸ ⊢⊣ ⊦⫞ ⊨⫤ ⊩⫣ ⊫⫥ ⊰⊱ ⊲⊳ ⊴⊵ ⊶⊷ ⋉⋊ ⋋⋌ ⋐⋑ ⋖⋗ ⋘⋙ ⋚⋛
|
|
261
|
+
⋜⋝ ⋞⋟ ⋠⋡ ⋢⋣ ⋤⋥ ⋦⋧ ⋨⋩ ⋪⋫ ⋬⋭ ⋰⋱ ⋲⋺ ⋳⋻ ⋴⋼ ⋶⋽ ⋷⋾ ⌈⌉ ⌊⌋ 〈〉 ⎴⎵ ❨❩ ❪❫
|
|
262
|
+
❬❭ ❮❯ ❰❱ ❲❳ ❴❵ ⟃⟄ ⟅⟆ ⟕⟖ ⟝⟞ ⟢⟣ ⟤⟥ ⟦⟧ ⟨⟩ ⟪⟫ ⦃⦄ ⦅⦆ ⦇⦈ ⦉⦊ ⦋⦌ ⦍⦎ ⦏⦐
|
|
263
|
+
⦑⦒ ⦓⦔ ⦕⦖ ⦗⦘ ⧀⧁ ⧄⧅ ⧏⧐ ⧑⧒ ⧔⧕ ⧘⧙ ⧚⧛ ⧸⧹ ⧼⧽ ⨫⨬ ⨭⨮ ⨴⨵ ⨼⨽ ⩤⩥ ⩹⩺ ⩽⩾ ⩿⪀
|
|
264
|
+
⪁⪂ ⪃⪄ ⪋⪌ ⪑⪒ ⪓⪔ ⪕⪖ ⪗⪘ ⪙⪚ ⪛⪜ ⪡⪢ ⪦⪧ ⪨⪩ ⪪⪫ ⪬⪭ ⪯⪰ ⪳⪴ ⪻⪼ ⪽⪾ ⪿⫀ ⫁⫂ ⫃⫄
|
|
265
|
+
⫅⫆ ⫍⫎ ⫏⫐ ⫑⫒ ⫓⫔ ⫕⫖ ⫬⫭ ⫷⫸ ⫹⫺ ⸂⸃ ⸄⸅ ⸉⸊ ⸌⸍ ⸜⸝ ⸠⸡ 〈〉 《》 「」 『』 【】 〔〕
|
|
266
|
+
〖〗 〘〙 〚〛 〝〞 ﴾﴿ ︗︘ ︵︶ ︷︸ ︹︺ ︻︼ ︽︾ ︿﹀ ﹁﹂ ﹃﹄ ﹇﹈ ﹙﹚ ﹛﹜ ﹝﹞ () <> []
|
|
267
|
+
{} ⦅⦆ 「」
|
|
268
|
+
).to_h(&:chars).freeze
|
|
269
|
+
|
|
270
|
+
# --- building blocks for the rules below
|
|
271
|
+
|
|
272
|
+
# Ruby's \w, \d and \s are ASCII-only. Raku is not, so the rules use
|
|
273
|
+
# \p{Nd} and \p{Space}, and word characters are spelled out.
|
|
274
|
+
w = '\p{L}\p{N}_'
|
|
275
|
+
# Superscript digits and signs are exponents ("$x²"), so they must
|
|
276
|
+
# not be part of an identifier.
|
|
277
|
+
sup = '²³¹⁰-⁻'
|
|
278
|
+
word = "[#{w}&&[^#{sup}]]"
|
|
279
|
+
nondigit = "[\\p{L}\\p{No}\\p{Nl}_&&[^#{sup}]]"
|
|
280
|
+
# one identifier character, for lookbehinds
|
|
281
|
+
ident_char = "[#{w}'\\-:&&[^#{sup}]]"
|
|
282
|
+
# What may follow a word for it to still be part of a longer
|
|
283
|
+
# identifier. A single colon does not continue an identifier
|
|
284
|
+
# ("@a.push: 1", "Blob:D:"), and neither does a hyphen that is not
|
|
285
|
+
# followed by a letter ("$x-1").
|
|
286
|
+
ident_end = "(?:#{word}|['\\-](?=[\\p{L}\\p{No}\\p{Nl}_])|::)"
|
|
287
|
+
# not preceded by something that makes this the tail of a longer word
|
|
288
|
+
nw = "(?<![#{w}'\\-])"
|
|
289
|
+
|
|
290
|
+
# an identifier; a hyphen or apostrophe must be followed by a letter
|
|
291
|
+
ident = "#{nondigit}#{word}*(?:['\\-]#{nondigit}#{word}*)*"
|
|
292
|
+
qualified_ident = "#{ident}(?:::#{ident})*"
|
|
293
|
+
# the operator part of a routine name: infix:<+>, circumfix:«[ ]»
|
|
294
|
+
op_name_suffix = '(?::(?:sym)?(?:<[^>\n]+>|«[^»\n]+»|' \
|
|
295
|
+
'\[(?:"[^"\n]*"|\'[^\'\n]*\'|[^\]\n])+\]))'
|
|
296
|
+
routine_name = "[!^]?#{qualified_ident}#{op_name_suffix}?"
|
|
297
|
+
op_categories = '(?:infix|prefix|postfix|circumfix|postcircumfix|term|trait_mod)'
|
|
298
|
+
type_name = "[A-Z]#{word}*(?:['\\-]#{nondigit}#{word}*)*(?:::#{ident})*"
|
|
299
|
+
# type smileys: Int:D, Foo:U, Any:_
|
|
300
|
+
smiley = "(?::[UD_](?![#{w}'\\-]))"
|
|
301
|
+
after_dot = '(?:(?<=\.)(?<!\.\.)|(?<=\.[\^?&+*]))'
|
|
302
|
+
subscript = '(?:\[[^\]\n]*\]|\{[^}\n]*\}|<[^>\n]*>)'
|
|
303
|
+
# the arguments of a call, with one level of nested parentheses;
|
|
304
|
+
# a parenthesis in a quoted string does not count
|
|
305
|
+
# (atomic and possessive, so that it can not backtrack for ages)
|
|
306
|
+
arguments = '\((?>\'[^\'\n]*\'|"[^"\n]*"|[^()\n]|\([^()\n]*\))*+\)'
|
|
307
|
+
# what can follow a variable in a string: a subscript or a method call
|
|
308
|
+
postfix = "(?:#{subscript}|\\.[\\^?&]?#{ident}#{arguments})"
|
|
309
|
+
# %h<key>, %h<<$key>>, %h«key»; kept to one line so that a stray '<'
|
|
310
|
+
# in code can't swallow what follows it
|
|
311
|
+
angle_subscripts = '(?:<<[^>\n]*>>|<[^>\n]*>|«[^»\n]*»)*'
|
|
312
|
+
|
|
313
|
+
# Where a "/" starts a regex rather than being a division: after an
|
|
314
|
+
# operator or an opening bracket, or after a word that takes a term.
|
|
315
|
+
regex_words = %w(
|
|
316
|
+
say put print when if elsif unless while until given with without
|
|
317
|
+
and or not so return grep map first split comb match subst
|
|
318
|
+
contains ff fff xor andthen orelse
|
|
319
|
+
)
|
|
320
|
+
regex_position = "(?=\\p{Space}*\\/)(?:(?<=[=(,{\\[;:!~|&?])|(?<==>)|" \
|
|
321
|
+
"#{regex_words.map { |x| "(?<=\\b#{x} )" }.join('|')})"
|
|
322
|
+
open_brackets = Regexp.escape(BRACKETS.keys.join)
|
|
323
|
+
|
|
324
|
+
# Where q, m, s and the like can start a quote: not after a dot, where
|
|
325
|
+
# they are method names ("$file.s / 1024").
|
|
326
|
+
quote_start = "(?<![#{w}'\\-.])"
|
|
327
|
+
# the adverbs of a quote: q:to, qq:!c, m:i:g
|
|
328
|
+
adverbs = "(?:\\p{Space}*:!?[#{w}]+)+"
|
|
329
|
+
# Between a quote word and its delimiter, whitespace can only come
|
|
330
|
+
# before a bracket or a slash: in "q ~~ $x" and "m ?? 1 !! 2", q and
|
|
331
|
+
# m are names.
|
|
332
|
+
before_delimiter = "(?:\\p{Space}*(?=[#{open_brackets}\\/])|)"
|
|
333
|
+
# where a "<" starts a word list rather than being a comparison
|
|
334
|
+
term_position = "(?=\\p{Space}*<)(?:(?<=[=(,\\[])|(?<=\\bfor ))"
|
|
335
|
+
# "token", "rule" and "regex" are ordinary words in "$x.rule",
|
|
336
|
+
# "rule => 1" or "/regex/"
|
|
337
|
+
not_a_regex_declarator = "(?<![#{w}'.:$@%&\\/<\\-])"
|
|
338
|
+
|
|
339
|
+
# A regex matching the longest of +words+. Common prefixes are shared,
|
|
340
|
+
# so that it does not slow down as the list grows.
|
|
341
|
+
any_of = lambda do |words|
|
|
342
|
+
tree = {}
|
|
343
|
+
words.each do |x|
|
|
344
|
+
x.each_char.inject(tree) { |node, char| node[char] ||= {} }[:end] = true
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
build = lambda do |node|
|
|
348
|
+
branches = node.reject { |char, _| char == :end }.map do |char, rest|
|
|
349
|
+
Regexp.escape(char) + build.(rest)
|
|
350
|
+
end
|
|
351
|
+
next '' if branches.empty?
|
|
352
|
+
next branches.first if branches.length == 1 && !node[:end]
|
|
353
|
+
|
|
354
|
+
"(?:#{branches.join('|')})#{'?' if node[:end]}"
|
|
355
|
+
end
|
|
356
|
+
build.(tree)
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
word_match = lambda do |words, suffix = ''|
|
|
360
|
+
"(?<!#{ident_char})(#{any_of.(words)})" \
|
|
361
|
+
"#{suffix}(?!#{ident_end})"
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
not_a_builtin_type = "(?!(?:#{any_of.(BUILTIN_CLASSES)}|True|False|Nil)" \
|
|
365
|
+
"#{smiley}?(?!#{ident_end}))"
|
|
366
|
+
# After a dot every builtin routine is a builtin, and so is the name
|
|
367
|
+
# of a type: "$x.Int" calls the coercion method, it does not name
|
|
368
|
+
# the type.
|
|
369
|
+
method_builtins = (BUILTINS | METHODS | BUILTIN_CLASSES).sort
|
|
370
|
+
builtins = Set.new(BUILTINS)
|
|
371
|
+
|
|
372
|
+
OPEN_BRACKET = /[#{open_brackets}]/
|
|
373
|
+
HEREDOC_OPENER = /#{nw}(qq|q|Q)[a-zA-Z]?\p{Space}*((?::[#{w}]+\p{Space}*)+)([^#{w}\p{Space}:])/
|
|
374
|
+
|
|
375
|
+
start do
|
|
376
|
+
@brace_levels = []
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# Consumes the rest of a construct opened with the delimiter +opener+
|
|
380
|
+
# and returns its body and its closing delimiter. Mirrored delimiters
|
|
381
|
+
# nest. An unclosed construct runs to the end of the input, and has
|
|
382
|
+
# an empty closing delimiter.
|
|
383
|
+
def scan_delimited(stream, opener, escapes: false)
|
|
384
|
+
mirror = BRACKETS[opener[0]]
|
|
385
|
+
closer = mirror ? mirror * opener.length : opener
|
|
386
|
+
patterns = [closer, opener].uniq.map { |x| Regexp.escape(x) }
|
|
387
|
+
# a backslash hides the delimiter that follows it
|
|
388
|
+
patterns << '\\\\.' if escapes
|
|
389
|
+
delimiters = /#{patterns.join('|')}/m
|
|
390
|
+
|
|
391
|
+
body = +''
|
|
392
|
+
depth = 1
|
|
393
|
+
loop do
|
|
394
|
+
chunk = stream.scan_until(delimiters)
|
|
395
|
+
if chunk.nil?
|
|
396
|
+
body << stream.rest
|
|
397
|
+
stream.terminate
|
|
398
|
+
return [body, '']
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
body << chunk
|
|
402
|
+
case stream.matched
|
|
403
|
+
when closer then depth -= 1
|
|
404
|
+
when opener then depth += 1
|
|
405
|
+
end
|
|
406
|
+
return [body[0...-closer.length], closer] if depth.zero?
|
|
407
|
+
end
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
# Consumes the rest of a regex (or of the replacement of a
|
|
411
|
+
# substitution) opened with the delimiter +opener+ and returns its
|
|
412
|
+
# body and its closing delimiter. Unlike #scan_delimited this steps
|
|
413
|
+
# over quoted strings, so the delimiter may appear in them:
|
|
414
|
+
# rx/ '/tmp/' .* /
|
|
415
|
+
def scan_regex(stream, opener, match_variable: false)
|
|
416
|
+
mirror = BRACKETS[opener[0]]
|
|
417
|
+
closer = mirror ? mirror * opener.length : opener
|
|
418
|
+
closing = /#{Regexp.escape(closer)}/
|
|
419
|
+
opening = mirror && /#{Regexp.escape(opener)}/
|
|
420
|
+
plain = /[^\\'"$#{Regexp.escape((opener[0] + closer[0]).squeeze)}]+/
|
|
421
|
+
# only a quote that is closed on the same line counts
|
|
422
|
+
quoted = /'(?:\\.|[^'\\\n])*'|"(?:\\.|[^"\\\n])*"/
|
|
423
|
+
# "$/" in the replacement of s/.../.../ is the match variable,
|
|
424
|
+
# provided the real delimiter still follows on this line
|
|
425
|
+
variable = match_variable && /\$#{closing}(?=[^\n]*#{closing})/
|
|
426
|
+
|
|
427
|
+
body = +''
|
|
428
|
+
depth = 1
|
|
429
|
+
until stream.eos?
|
|
430
|
+
if (text = stream.scan(plain) || stream.scan(/\\./m) || stream.scan(quoted) ||
|
|
431
|
+
(variable && stream.scan(variable)))
|
|
432
|
+
body << text
|
|
433
|
+
elsif stream.scan(closing)
|
|
434
|
+
depth -= 1
|
|
435
|
+
return [body, closer] if depth.zero?
|
|
436
|
+
|
|
437
|
+
body << closer
|
|
438
|
+
elsif opening && stream.scan(opening)
|
|
439
|
+
depth += 1
|
|
440
|
+
body << opener
|
|
441
|
+
else
|
|
442
|
+
body << stream.getch
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
[body, '']
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
# Lexes one delimited part of a regex-like construct, whose opening
|
|
449
|
+
# delimiter has just been consumed. The body is lexed in +state+,
|
|
450
|
+
# or is a plain token if there is none.
|
|
451
|
+
def lex_regex_part(stream, opener, state, match_variable: false)
|
|
452
|
+
body, closing = scan_regex(stream, opener, match_variable: match_variable)
|
|
453
|
+
if state
|
|
454
|
+
sublex body, state
|
|
455
|
+
else
|
|
456
|
+
token Str::Regex, body
|
|
457
|
+
end
|
|
458
|
+
token Str::Regex, closing
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# s/pattern/replacement/, s{pattern}{replacement}, tr/from/to/, ...
|
|
462
|
+
def lex_substitution(stream, opening, kind, opener)
|
|
463
|
+
transliteration = kind.downcase == 'tr'
|
|
464
|
+
|
|
465
|
+
token Str::Regex, opening
|
|
466
|
+
lex_regex_part(stream, opener, transliteration ? nil : :regex_body)
|
|
467
|
+
|
|
468
|
+
replacement = transliteration ? nil : :interpolated
|
|
469
|
+
if BRACKETS[opener[0]].nil?
|
|
470
|
+
# s/a/b/ : the replacement follows directly, with the same delimiter
|
|
471
|
+
lex_regex_part(stream, opener, replacement, match_variable: true)
|
|
472
|
+
elsif stream.check(/\p{Space}*#{OPEN_BRACKET}/)
|
|
473
|
+
# s{a}{b} : another bracketed group, possibly after whitespace.
|
|
474
|
+
# Otherwise it is s{a} = b, an ordinary assignment.
|
|
475
|
+
token Text::Whitespace, stream.scan(/\p{Space}*/)
|
|
476
|
+
opener = stream.scan(/(.)\1*/m)
|
|
477
|
+
token Str::Regex, opener
|
|
478
|
+
lex_regex_part(stream, opener, replacement)
|
|
479
|
+
end
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# Lexes +text+ on its own, starting in +state+.
|
|
483
|
+
def sublex(text, state = :root)
|
|
484
|
+
lexer = self.class.new(options)
|
|
485
|
+
lexer.reset!
|
|
486
|
+
lexer.push(state) unless state == :root
|
|
487
|
+
delegate(lexer, text)
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
# Lexes what follows the opening quote of a heredoc: the rest of
|
|
491
|
+
# that line, which is ordinary code, and then the heredoc's body.
|
|
492
|
+
def lex_heredocs(stream, terminator, interpolate)
|
|
493
|
+
rest_of_line = stream.scan(/[^\n]*/)
|
|
494
|
+
sublex rest_of_line
|
|
495
|
+
return unless stream.scan(/\n/)
|
|
496
|
+
|
|
497
|
+
token Text::Whitespace, "\n"
|
|
498
|
+
|
|
499
|
+
# the bodies follow in the order their heredocs were opened
|
|
500
|
+
heredocs = [[terminator, interpolate]] + extra_heredocs(rest_of_line)
|
|
501
|
+
heredocs.each_with_index do |(name, interpolates), index|
|
|
502
|
+
last_line = /^[ \t]*#{Regexp.escape(name)}[ \t]*$/
|
|
503
|
+
body = stream.scan_until(/(?=#{last_line})/)
|
|
504
|
+
if body.nil?
|
|
505
|
+
body = stream.rest
|
|
506
|
+
stream.terminate
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
if interpolates
|
|
510
|
+
sublex body, :interpolated
|
|
511
|
+
else
|
|
512
|
+
token Str, body
|
|
513
|
+
end
|
|
514
|
+
break if stream.eos?
|
|
515
|
+
|
|
516
|
+
token Str, stream.scan(last_line)
|
|
517
|
+
if index + 1 < heredocs.length && stream.scan(/\n/)
|
|
518
|
+
token Text::Whitespace, "\n"
|
|
519
|
+
end
|
|
520
|
+
end
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# Heredocs opened later on the same line, as pairs of terminator
|
|
524
|
+
# and whether the body interpolates.
|
|
525
|
+
def extra_heredocs(line)
|
|
526
|
+
found = []
|
|
527
|
+
pos = 0
|
|
528
|
+
while (match = HEREDOC_OPENER.match(line, pos))
|
|
529
|
+
pos = match.end(0)
|
|
530
|
+
quote, adverbs, opener = match.captures
|
|
531
|
+
next unless adverbs.match?(/:to\b/)
|
|
532
|
+
|
|
533
|
+
stop = line.index(BRACKETS.fetch(opener, opener), pos)
|
|
534
|
+
next unless stop
|
|
535
|
+
|
|
536
|
+
found << [line[pos...stop], quote == 'qq' || adverbs.include?(':qq')]
|
|
537
|
+
end
|
|
538
|
+
found
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
# If you're modifying these rules, be careful if you need to process
|
|
542
|
+
# '{' or '}' characters. Code can be nested in strings, and the
|
|
543
|
+
# :embedded state counts braces to know where that code ends, so if
|
|
544
|
+
# you process one of them, make sure you also process the other!
|
|
545
|
+
state :common do
|
|
546
|
+
# --- Pod and regexes. These can start with whitespace, which the
|
|
547
|
+
# rules after them skip.
|
|
548
|
+
# Pod: everything after =finish, a delimited block (=begin pod ...
|
|
549
|
+
# =end pod), or a paragraph or abbreviated block, which ends at the
|
|
550
|
+
# first blank line
|
|
551
|
+
rule %r/^=finish\b.*|^(\p{Space}*)=begin\p{Space}+([#{w}]+)\b.*?^\1=end\p{Space}+\2|^\p{Space}*=for.*?\n\p{Space}*?\n|^=.*?\n\p{Space}*?\n/m do |m|
|
|
552
|
+
sublex m[0], :pod_body
|
|
553
|
+
end
|
|
554
|
+
# a regex without m or rx: $s ~~ /x/, .subst(/x/, ''), say /x/
|
|
555
|
+
rule %r/#{regex_position}(\p{Space}*)(\/)(?!\/)(?=(?:\\[\s\S]|[^\/\\\n])*\/)/ do |m|
|
|
556
|
+
opener = m[2]
|
|
557
|
+
groups Text::Whitespace, Str::Regex
|
|
558
|
+
lex_regex_part(m, opener, :regex_body)
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# a word list where a term is expected, which unlike the general
|
|
562
|
+
# rule for <a b c> further down can start with a space, span lines
|
|
563
|
+
# and hold any character: my @a = < a " b >;
|
|
564
|
+
rule %r/#{term_position}(\p{Space}*)(<(?!<|=?\])\p{Space}[^<>]*>)/ do
|
|
565
|
+
groups Text::Whitespace, Str
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
# --- whitespace and punctuation. No rule below starts with either,
|
|
569
|
+
# and they are the most common tokens, so they are not made to
|
|
570
|
+
# wait for all the other rules to fail first.
|
|
571
|
+
rule %r/\p{Space}+/, Text::Whitespace
|
|
572
|
+
rule %r/[;,()\]]/, Punctuation
|
|
573
|
+
|
|
574
|
+
# --- comments
|
|
575
|
+
rule %r/#[`|=](([#{open_brackets}])\2*)/ do |m|
|
|
576
|
+
opening = m[0]
|
|
577
|
+
token Comment::Multiline, opening + scan_delimited(m, m[1]).join
|
|
578
|
+
end
|
|
579
|
+
rule %r/#[|=].*/, Comment::Special
|
|
580
|
+
rule %r/#.*/, Comment::Single
|
|
581
|
+
|
|
582
|
+
# --- variables. Nothing else starts with a sigil either.
|
|
583
|
+
# attributes ($!x, $.x) and compile-time / pod variables ($?FILE, $=pod)
|
|
584
|
+
rule %r/[$@%&][.!]#{qualified_ident}#{angle_subscripts}/, Name::Variable::Instance
|
|
585
|
+
rule %r/[$@%&][?=]#{ident}#{angle_subscripts}/, Name::Variable::Magic
|
|
586
|
+
rule %r/::\?[#{w}]+/, Name::Variable::Global
|
|
587
|
+
rule %r/[$@%&]\*#{qualified_ident}#{angle_subscripts}/, Name::Variable::Global
|
|
588
|
+
rule %r/\$[!\/¢]#{angle_subscripts}/, Name::Variable::Global
|
|
589
|
+
# the & sigil names a routine: &infix:<+>, &squared, &squared(2),
|
|
590
|
+
# which may be one that comes with the language: &elems
|
|
591
|
+
rule %r/&#{op_categories}#{op_name_suffix}/, Name::Function
|
|
592
|
+
rule %r/&(?:::)?(#{qualified_ident})/ do |m|
|
|
593
|
+
token builtins.include?(m[1]) ? Name::Builtin : Name::Function
|
|
594
|
+
end
|
|
595
|
+
rule %r/[$@%&][\^:~]?(?:::)?(?:#{qualified_ident}|\p{Nd}+)#{angle_subscripts}/, Name::Variable
|
|
596
|
+
rule %r/\$(?:<[^>\n]*>)+/, Name::Variable
|
|
597
|
+
# anonymous variables: "state $ = 0", "$++"
|
|
598
|
+
rule %r/[$@](?=[\p{Space}=;,)\]]|\+\+|--)/, Name::Variable
|
|
599
|
+
rule %r/%(?=[,)])/, Name::Variable
|
|
600
|
+
# contextualizers: $(...), @(...), @$x, $@a
|
|
601
|
+
rule %r/[$@](?=[(\[{])/, Operator
|
|
602
|
+
rule %r/[$@%&](?=[$@%&][#{w}.!*?^])/, Operator
|
|
603
|
+
# sigilless variables (\x), capture literals \(1, 2) and unspace
|
|
604
|
+
rule %r/\\#{ident}/, Name::Variable
|
|
605
|
+
rule %r/\\(?=[\p{Space}(])/, Operator
|
|
606
|
+
# type captures: ::T
|
|
607
|
+
rule %r/::#{ident}/, Name::Class
|
|
608
|
+
|
|
609
|
+
# --- numbers. Nothing else starts with a digit.
|
|
610
|
+
rule %r/0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
|
|
611
|
+
rule %r/0o[0-7]+(?:_[0-7]+)*/, Num::Oct
|
|
612
|
+
rule %r/0b[01]+(?:_[01]+)*/, Num::Bin
|
|
613
|
+
rule %r/0d\p{Nd}+(?:_\p{Nd}+)*/, Num::Integer
|
|
614
|
+
# radix literals: :16<FF>
|
|
615
|
+
rule %r/:\p{Nd}+<[0-9a-z_.]+>/i, Num
|
|
616
|
+
# imaginary numbers
|
|
617
|
+
rule %r/(?:\p{Nd}+(?:_\p{Nd}+)*(?:\.\p{Nd}+(?:_\p{Nd}+)*)?|\.\p{Nd}+(?:_\p{Nd}+)*)(?:e[+-]?\p{Nd}+)?i(?![#{w}'\-])/i, Num::Float
|
|
618
|
+
rule %r/(?:\p{Nd}+(?:_\p{Nd}+)*)?\.\p{Nd}+(?:_\p{Nd}+)*(?:e[+-]?\p{Nd}+)?/i, Num::Float
|
|
619
|
+
rule %r/\p{Nd}+(?:_\p{Nd}+)*e[+-]?\p{Nd}+/i, Num::Float
|
|
620
|
+
rule %r/\p{Nd}+(?:_\p{Nd}+)*/, Num::Integer
|
|
621
|
+
# rational and complex literals: <1/3>, <1+2i>
|
|
622
|
+
rule %r/<[-+]?\p{Nd}+\/\p{Nd}+>/, Num
|
|
623
|
+
rule %r/<[-+]?[\p{Nd}.]+[-+][\p{Nd}.]+i>/, Num
|
|
624
|
+
|
|
625
|
+
# --- regex declarations, only when a name or a block follows
|
|
626
|
+
rule %r/#{not_a_regex_declarator}(regex|token|rule)(\p{Space}+)(#{ident}:sym)/ do
|
|
627
|
+
groups Keyword::Declaration, Text::Whitespace, Name::Function
|
|
628
|
+
push :token_sym_brackets
|
|
629
|
+
end
|
|
630
|
+
rule %r/#{not_a_regex_declarator}(regex|token|rule)(?=\p{Space}+[\p{L}\p{No}\p{Nl}_]|\p{Space}*\{)(?!\p{Space}+[#{w}]+\p{Space}*=>)(\p{Space}*)(#{qualified_ident})?/ do
|
|
631
|
+
groups Keyword::Declaration, Text::Whitespace, Name::Function
|
|
632
|
+
push :pre_token
|
|
633
|
+
end
|
|
634
|
+
|
|
635
|
+
# deal with a special case in the Raku grammar (role q { ... })
|
|
636
|
+
rule %r/(role)(\p{Space}+)(q)(\p{Space}*)/ do
|
|
637
|
+
groups Keyword::Declaration, Text::Whitespace, Name, Text::Whitespace
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
# --- quote-like constructs: q/raw/, qq{interpolating}, Q[literal].
|
|
641
|
+
# A bracket can be repeated to make a longer delimiter (q<< >>);
|
|
642
|
+
# other characters can not, so qq|| is an empty string.
|
|
643
|
+
# and heredocs (q:to/END/). Before the keyword and builtin rules,
|
|
644
|
+
# which would otherwise take q for a word.
|
|
645
|
+
rule %r/#{quote_start}(qq|q|Q)[a-zA-Z]?(?:(#{adverbs})\p{Space}*|#{before_delimiter})((#{OPEN_BRACKET})\4*|[^0-9a-zA-Z:\p{Space}=,;)])/ do |m|
|
|
646
|
+
opening = m[0]
|
|
647
|
+
adverbs = m[2].to_s
|
|
648
|
+
# qq strings (and the :qq / :c adverbs) interpolate
|
|
649
|
+
interpolate = m[1] == 'qq' || adverbs.match?(/:(?:qq|c)\b/)
|
|
650
|
+
# nothing is special in Q, not even a backslash: Q/\/
|
|
651
|
+
body, closing = scan_delimited(m, m[3], escapes: m[1] != 'Q')
|
|
652
|
+
|
|
653
|
+
if adverbs.match?(/:to\b/)
|
|
654
|
+
token Str, opening + body + closing
|
|
655
|
+
lex_heredocs(m, body, interpolate)
|
|
656
|
+
elsif interpolate
|
|
657
|
+
token Str, opening
|
|
658
|
+
sublex body, :interpolated
|
|
659
|
+
token Str, closing
|
|
660
|
+
else
|
|
661
|
+
token Str, opening + body + closing
|
|
662
|
+
end
|
|
663
|
+
end
|
|
664
|
+
|
|
665
|
+
# --- regexes: m/x/, rx{x}, and with adverbs m:i/x/
|
|
666
|
+
rule %r/#{quote_start}(?:m|ms|rx)(?:#{adverbs}\p{Space}*|#{before_delimiter})((#{OPEN_BRACKET})\2*|[^#{w}:\p{Space}=,;)])/ do |m|
|
|
667
|
+
opener = m[1]
|
|
668
|
+
token Str::Regex
|
|
669
|
+
lex_regex_part(m, opener, :regex_body)
|
|
670
|
+
end
|
|
671
|
+
# substitution and transliteration: s/a/b/, S{a}{b}, s:2nd/a/b/, tr/a-z/A-Z/
|
|
672
|
+
# (a pattern can not be empty, so S|| and S%% are operators)
|
|
673
|
+
rule %r/#{quote_start}(ss|s|SS|S|tr|TR)(?=\p{Space}*:!?[#{w}])\p{Space}*(?::!?[#{w}\-]+(?:\([^)\n]*\))?\p{Space}*)+((#{OPEN_BRACKET})\3*|[^#{w}:\p{Space}$@%&=,;)])/ do |m|
|
|
674
|
+
lex_substitution(m, m[0], m[1], m[2])
|
|
675
|
+
end
|
|
676
|
+
rule %r/#{quote_start}(ss|s|SS|S|tr|TR)#{before_delimiter}(([{(\[])\3*|([\/|!^~@%])(?!\4))/ do |m|
|
|
677
|
+
lex_substitution(m, m[0], m[1], m[2])
|
|
678
|
+
end
|
|
679
|
+
# --- curly and corner quotes: ‘raw’ (or ‚this‘, or ’this‘),
|
|
680
|
+
# “interpolating” (or „this“, or ”this“), 「no escapes」
|
|
681
|
+
rule %r/[‘‚’][^‘’]*[’‘]/, Str::Single
|
|
682
|
+
rule %r/「[^」]*」/, Str
|
|
683
|
+
rule %r/[“„”]/, Str::Double, :dq_curly
|
|
684
|
+
|
|
685
|
+
# --- names whose meaning depends on where they are
|
|
686
|
+
# method calls: .say, .Int, .^name, .?foo
|
|
687
|
+
rule %r/#{after_dot}#{word_match.(method_builtins)}/, Name::Builtin
|
|
688
|
+
rule %r/#{after_dot}#{ident}/, Name::Function
|
|
689
|
+
# the key of a pair is a plain word, whatever it spells: name => 1
|
|
690
|
+
rule %r/(?<!#{ident_char})#{qualified_ident}(?=\p{Space}*=>)/, Name
|
|
691
|
+
# operators used by name: infix:<+>(1, 2)
|
|
692
|
+
rule %r/(?<!#{ident_char})#{op_categories}#{op_name_suffix}/, Name::Function
|
|
693
|
+
# traits: is rw, is copy, is export
|
|
694
|
+
rule %r/(?<!#{ident_char})(is)(\p{Space}+)#{word_match.(TRAITS)}/ do
|
|
695
|
+
groups Keyword, Text::Whitespace, Keyword
|
|
696
|
+
end
|
|
697
|
+
# user-defined types: "is Foo", "of Foo", "--> Foo"
|
|
698
|
+
rule %r/(?<!#{ident_char})(is|does|of|returns|handles|trusts|hides)(\p{Space}+)#{not_a_builtin_type}(#{type_name}#{smiley}?)(?!#{ident_end})/ do
|
|
699
|
+
groups Keyword, Text::Whitespace, Name::Class
|
|
700
|
+
end
|
|
701
|
+
rule %r/(-->)(\p{Space}*)#{not_a_builtin_type}(#{type_name}#{smiley}?)(?!#{ident_end})/ do
|
|
702
|
+
groups Operator, Text::Whitespace, Name::Class
|
|
703
|
+
end
|
|
704
|
+
# version literals: use v6.d; use v6.e.PREVIEW; use v6.d+;
|
|
705
|
+
rule %r/#{nw}(use|need|require)(\p{Space}+)(v\p{Nd}+(?:\.(?:\p{Nd}+|\*|[A-Za-z]+))*\+?)(?![#{w}'\-])/ do
|
|
706
|
+
groups Keyword::Namespace, Text::Whitespace, Num
|
|
707
|
+
end
|
|
708
|
+
# module names: use Foo::Bar; need Baz; (pragmas such as 'use lib' too)
|
|
709
|
+
rule %r/#{nw}(use|need|require|import|no)(\p{Space}+)(#{qualified_ident})(?!#{ident_end})/ do
|
|
710
|
+
groups Keyword::Namespace, Text::Whitespace, Name::Namespace
|
|
711
|
+
end
|
|
712
|
+
|
|
713
|
+
# --- declarations
|
|
714
|
+
rule %r/#{word_match.(%w(class role grammar module package knowhow enum subset))}(\p{Space}+)(#{qualified_ident})/ do
|
|
715
|
+
groups Keyword::Declaration, Text::Whitespace, Name::Class
|
|
716
|
+
end
|
|
717
|
+
rule %r/#{word_match.(%w(sub method submethod macro))}(\p{Space}+)(#{routine_name})/ do
|
|
718
|
+
groups Keyword::Declaration, Text::Whitespace, Name::Function
|
|
719
|
+
end
|
|
720
|
+
# "sub" is optional after multi, proto and only: multi foo(Int $x) { }
|
|
721
|
+
rule %r/#{word_match.(%w(multi proto only))}(\p{Space}+)(?!(?:sub|method|submethod|token|rule|regex|macro)(?!#{ident_end}))(#{routine_name})(?=\p{Space}*[({])/ do
|
|
722
|
+
groups Keyword::Declaration, Text::Whitespace, Name::Function
|
|
723
|
+
end
|
|
724
|
+
|
|
725
|
+
# --- keywords and other reserved words
|
|
726
|
+
rule %r/#{word_match.(DECLARATORS)}/, Keyword::Declaration
|
|
727
|
+
rule %r/#{word_match.(NAMESPACE_KEYWORDS)}/, Keyword::Namespace
|
|
728
|
+
rule %r/#{word_match.(KEYWORDS)}/, Keyword
|
|
729
|
+
rule %r/#{word_match.(%w(True False Nil))}/, Keyword::Constant
|
|
730
|
+
rule %r/#{word_match.(%w(self))}/, Name::Builtin::Pseudo
|
|
731
|
+
rule %r/#{word_match.(CONSTANTS)}/, Name::Constant
|
|
732
|
+
rule %r/[∞∅]/, Name::Constant
|
|
733
|
+
# version literals: v6.d, v1.2.3, v1.2+
|
|
734
|
+
rule %r/(?<!#{ident_char})v\p{Nd}+(?:\.(?:\p{Nd}+|\*|[a-z]))*\+?(?!#{ident_end})/, Num
|
|
735
|
+
# meta operators: Z+, X~, R-, Z=>, Rcmp, and x=, xx=
|
|
736
|
+
rule %r/#{nw}[RXZ](?:\*\*|\/\/|&&|\|\||<=>|=>|==|!=|<=|>=|~~|[-+*\/%~,&|^?<>]|(?:cmp|eq|ne|lt|gt|le|ge|leg|eqv|min|max|div|mod|and|or|xor|x|xx)(?![#{w}'\-]))/, Operator
|
|
737
|
+
rule %r/#{nw}(?:xx?|min|max)=(?![=~>])/, Operator
|
|
738
|
+
rule %r/#{word_match.(WORD_OPERATORS, '(?!\()')}/, Operator::Word
|
|
739
|
+
|
|
740
|
+
# --- types and routines that come with the language
|
|
741
|
+
# exception and AST class families: X::AdHoc, CX::Done, RakuAST::Name...
|
|
742
|
+
rule %r/(?<![#{w}':\-])(?:X|CX|RakuAST)(?:::[A-Za-z][#{w}'\-]*)+#{smiley}?(?![#{w}'\-])/, Keyword::Type
|
|
743
|
+
rule %r/#{word_match.(BUILTIN_CLASSES, "#{smiley}?")}/, Keyword::Type
|
|
744
|
+
rule %r/#{word_match.(BUILTINS)}/, Name::Builtin
|
|
745
|
+
# a user-defined type: with a smiley, or constraining a variable
|
|
746
|
+
rule %r/(?<!#{ident_char})#{type_name}#{smiley}/, Name::Class
|
|
747
|
+
rule %r/(?<!#{ident_char})#{type_name}(?=\p{Space}+[$@%&\\])/, Name::Class
|
|
748
|
+
rule %r/(?<=[#{w}>])#{smiley}/, Keyword::Type
|
|
749
|
+
|
|
750
|
+
# --- operators that could be mistaken for quotes
|
|
751
|
+
# hyper operators: »+«, >>+<<, +« (prefix), »++ and ».say (postfix)
|
|
752
|
+
rule %r/(?:«|»|<<|>>)[-+*\/%~=!&|^?<>]+(?:«|»|<<|>>)/, Operator
|
|
753
|
+
rule %r/[-+*\/%~!?|^]+«/, Operator
|
|
754
|
+
# an ASCII prefix hyper, when its operand follows: -<< (3, 2, 1)
|
|
755
|
+
rule %r/[-+~!?|^]<<(?=\p{Space}*[$@%(\[])/, Operator
|
|
756
|
+
# ... and around a bracketed operator: «[op]«, <<[op]>>. With a
|
|
757
|
+
# variable inside it is a subscript: @rows>>[$i]>>.chars
|
|
758
|
+
rule %r/(?:«|»|<<|>>)\[[^\]\n$@%]+\](?:«|»|<<|>>)/, Operator
|
|
759
|
+
rule %r/(?:»|>>)(?=\.[#{w}^?!(]|\+\+|--|[\[{<(])/, Operator
|
|
760
|
+
# reduction operators: [+], [\*], [<=], [max], [Z+]
|
|
761
|
+
rule %r/(?<![#{w}\])}>'\-])\[\\?(?:[RXZ]?[-+*\/%~=<>!&|^?,]+|[RXZ]?(?:min|max|gcd|lcm|and|or|xor|x|xx|cmp|eq|ne|lt|gt|le|ge|leg|eqv))\]/, Operator
|
|
762
|
+
# quote-like words with interpolation: «a $b» and <<a $b>>. After
|
|
763
|
+
# an operator or a term these are hyper operators, not quotes,
|
|
764
|
+
# and so they are in [«] and [<<], which name an operator.
|
|
765
|
+
rule %r/(?<![#{w}\])}>+\-*\/%~!?^|&.])«(?!\])/, Str::Double, :ww_guillemets
|
|
766
|
+
rule %r/(?<![#{w}\])}>])<<(?![=\]])/, Str::Double, :ww_angles
|
|
767
|
+
rule %r/(?!<->)<[^\p{Space}=<>{};()](?:[^<>{};()]*[^\p{Space}<>{};()])?>/, Str
|
|
768
|
+
|
|
769
|
+
# --- labels, pairs and operators
|
|
770
|
+
# loop labels: OUTER: for ...
|
|
771
|
+
rule %r/#{nw}([A-Z][A-Z0-9_]*)(:)(?=\p{Space}*(?:for|while|until|loop|repeat|given|if|unless|do|\{|$))/ do
|
|
772
|
+
groups Name::Label, Punctuation
|
|
773
|
+
end
|
|
774
|
+
# colon-pairs and adverbs: :name, :!flag, :name(...), :2nd
|
|
775
|
+
rule %r/(:!?)(#{ident})/ do
|
|
776
|
+
groups Punctuation, Name::Attribute
|
|
777
|
+
end
|
|
778
|
+
rule %r/(:)(\p{Nd}+)(#{ident})/ do
|
|
779
|
+
groups Punctuation, Num::Integer, Name::Attribute
|
|
780
|
+
end
|
|
781
|
+
# superscript exponents: $x², A⁻¹
|
|
782
|
+
rule %r/[#{sup}]+/, Operator
|
|
783
|
+
# a prefix operator in front of a function reference: ~&foo, +&bar
|
|
784
|
+
rule %r/[-~+?!|^](?=&[\p{L}\p{No}\p{Nl}_])/, Operator
|
|
785
|
+
# assignment meta operators: +=, //=, ||=, ...
|
|
786
|
+
rule %r/(?:\*\*|\/\/|\|\||&&|%%|[-+*\/%~|&^?])=(?![=~>])/, Operator
|
|
787
|
+
rule %r/#{any_of.(SYMBOL_OPERATORS)}/, Operator
|
|
788
|
+
# A name right before a parenthesis is a call: squared(2). Unless
|
|
789
|
+
# it is capitalized, as types are, which can be called to coerce:
|
|
790
|
+
# Foo($x)
|
|
791
|
+
rule %r/#{qualified_ident}(?=\()/ do |m|
|
|
792
|
+
token m[0].match?(/(?:\A|::)\p{Lu}[^:]*\z/) ? Name : Name::Function
|
|
793
|
+
end
|
|
794
|
+
rule %r/#{qualified_ident}/, Name
|
|
795
|
+
rule %r/'(?:\\\\|\\[^\\]|[^'\\])*'/, Str::Single
|
|
796
|
+
rule %r/"/, Str::Double, :dq_string
|
|
797
|
+
rule %r/[:\[]/, Punctuation
|
|
798
|
+
end
|
|
799
|
+
|
|
800
|
+
state :root do
|
|
801
|
+
mixin :common
|
|
802
|
+
rule %r/[{}]/, Punctuation
|
|
803
|
+
# anything else that is not a letter: user-defined operators, ...
|
|
804
|
+
rule %r/[^#{w}\p{Space}]/, Operator
|
|
805
|
+
rule %r/./m, Text
|
|
806
|
+
end
|
|
807
|
+
|
|
808
|
+
# between "token name" and the block that holds its body
|
|
809
|
+
state :pre_token do
|
|
810
|
+
# a statement that ends before any block was not a declaration
|
|
811
|
+
rule %r/;/, Punctuation, :pop!
|
|
812
|
+
mixin :common
|
|
813
|
+
rule %r/\{/ do
|
|
814
|
+
token Punctuation
|
|
815
|
+
goto :token
|
|
816
|
+
end
|
|
817
|
+
rule %r/[^#{w}\p{Space}]/, Operator
|
|
818
|
+
rule %r/./m, Text
|
|
819
|
+
end
|
|
820
|
+
|
|
821
|
+
# the bracketed part of a name: token infix:sym<+> { ... }
|
|
822
|
+
state :token_sym_brackets do
|
|
823
|
+
rule %r/(#{OPEN_BRACKET})\1*/ do |m|
|
|
824
|
+
opening = m[0]
|
|
825
|
+
token Name, opening + scan_delimited(m, opening).join
|
|
826
|
+
goto :pre_token
|
|
827
|
+
end
|
|
828
|
+
rule(//) { goto :pre_token }
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
state :token do
|
|
832
|
+
rule %r/\}/, Punctuation, :pop!
|
|
833
|
+
mixin :regex_body
|
|
834
|
+
end
|
|
835
|
+
|
|
836
|
+
# the body of a token, rule or regex, or of a m//, rx// or s///
|
|
837
|
+
# pattern
|
|
838
|
+
state :regex_body do
|
|
839
|
+
rule %r/\p{Space}+/, Text::Whitespace
|
|
840
|
+
rule %r/#.*/, Comment::Single
|
|
841
|
+
# :my $x = ...; declarations are ordinary code
|
|
842
|
+
rule %r/:(?=(?:my|our|state|constant|temp|let)\b)/, Punctuation
|
|
843
|
+
rule %r/(?<=:)(?:my|our|state|constant|temp|let).*?;/m do |m|
|
|
844
|
+
sublex m[0]
|
|
845
|
+
end
|
|
846
|
+
# adverbs: :i, :sigspace, :!ratchet, :Perl5(...)
|
|
847
|
+
rule %r/(:!?)([A-Za-z][#{w}\-]*)/ do
|
|
848
|
+
groups Punctuation, Name::Attribute
|
|
849
|
+
end
|
|
850
|
+
# character classes: <[a..z]>, <-[\p{Nd}] + [_]>
|
|
851
|
+
rule %r/<(?:[-+!?.]\p{Space}*)?\[(?:\\.|[^\]\\])*\](?:\p{Space}*[-+]\p{Space}*(?:\[(?:\\.|[^\]\\])*\]|:?[#{w}\-]+))*\p{Space}*>/m, Str::Regex
|
|
852
|
+
# unicode properties: <:Lu>, <+:L>, <:L + :N>, <:L - [a]>
|
|
853
|
+
rule %r/<[-+!?.]?\p{Space}*:[#{w}\-]+(?:\([^)\n]*\))?(?:\p{Space}*[-+]\p{Space}*(?::[#{w}\-]+|\[(?:\\.|[^\]\\])*\]))*\p{Space}*>/m, Name::Builtin
|
|
854
|
+
# named assertions: <foo>, <.foo>, <?before ...>, <name=rule>
|
|
855
|
+
rule %r/(<)([?!.+-]?)(\p{Space}*)(#{ident})(=)(#{ident})(>)/ do
|
|
856
|
+
groups Punctuation, Punctuation, Text::Whitespace, Name::Variable,
|
|
857
|
+
Operator, Name::Function, Punctuation
|
|
858
|
+
end
|
|
859
|
+
rule %r/(<)([?!.+-]?)(\p{Space}*)#{word_match.(REGEX_BUILTINS)}(>)?/ do
|
|
860
|
+
groups Punctuation, Punctuation, Text::Whitespace, Name::Builtin, Punctuation
|
|
861
|
+
end
|
|
862
|
+
rule %r/(<)([?!.+-]?)(\p{Space}*)(#{ident})(>)?/ do
|
|
863
|
+
groups Punctuation, Punctuation, Text::Whitespace, Name::Function, Punctuation
|
|
864
|
+
end
|
|
865
|
+
# code blocks and variables
|
|
866
|
+
rule %r/\{/ do
|
|
867
|
+
token Punctuation
|
|
868
|
+
@brace_levels << 1
|
|
869
|
+
push :embedded
|
|
870
|
+
end
|
|
871
|
+
rule %r/\$<[#{w}'\-]+>/, Name::Variable
|
|
872
|
+
rule %r/\$\p{Nd}+/, Name::Variable
|
|
873
|
+
rule %r/[$@][.^:?=!~*]?#{qualified_ident}#{angle_subscripts}/, Name::Variable
|
|
874
|
+
# literals
|
|
875
|
+
rule %r/'(?:\\.|[^'\\])*'/m, Str::Single
|
|
876
|
+
rule %r/[‘‚’][^‘’\n]*[’‘]/, Str::Single
|
|
877
|
+
rule %r/「[^」]*」/, Str
|
|
878
|
+
rule %r/"/, Str::Double, :dq_string
|
|
879
|
+
rule %r/[“„”]/, Str::Double, :dq_curly
|
|
880
|
+
rule %r/\\[xXcCoO]\[[^\]\n]*\]|\\x[0-9a-fA-F]+|\\./m, Str::Escape
|
|
881
|
+
# anchors, quantifiers, alternation, separators
|
|
882
|
+
rule %r/\^\^|\$\$|<<|>>|«|»|\^|\$/, Operator
|
|
883
|
+
rule %r/\*\*|\|\||&&|%%|\.\.\.?|::?:?|[|&*+?!%~=.]/, Operator
|
|
884
|
+
rule %r/[()\[\]<>]/, Punctuation
|
|
885
|
+
rule %r/\}/, Punctuation
|
|
886
|
+
rule %r/[#{w}]+/, Str::Regex
|
|
887
|
+
rule %r/./m, Str::Regex
|
|
888
|
+
end
|
|
889
|
+
|
|
890
|
+
# Code embedded in something else (a closure inside a string, or code
|
|
891
|
+
# inside a regex) is
|
|
892
|
+
# lexed in this state, which keeps count of its own braces so it
|
|
893
|
+
# knows when to hand control back.
|
|
894
|
+
state :embedded do
|
|
895
|
+
mixin :common
|
|
896
|
+
rule %r/\{/ do
|
|
897
|
+
token Punctuation
|
|
898
|
+
@brace_levels[-1] += 1
|
|
899
|
+
end
|
|
900
|
+
rule %r/\}/ do
|
|
901
|
+
token Punctuation
|
|
902
|
+
@brace_levels[-1] -= 1
|
|
903
|
+
if @brace_levels.last.zero?
|
|
904
|
+
@brace_levels.pop
|
|
905
|
+
pop!
|
|
906
|
+
end
|
|
907
|
+
end
|
|
908
|
+
rule %r/[^#{w}\p{Space}]/, Operator
|
|
909
|
+
rule %r/./m, Text
|
|
910
|
+
end
|
|
911
|
+
|
|
912
|
+
# Pod documentation blocks
|
|
913
|
+
state :pod_body do
|
|
914
|
+
rule %r/^(\p{Space}*)(=head\p{Nd}*)(.*)/ do
|
|
915
|
+
groups Text::Whitespace, Comment::Preproc, Generic::Heading
|
|
916
|
+
end
|
|
917
|
+
rule %r/^(\p{Space}*)(=(?:begin|end|for|finish))([ \t]*)([#{w}]*)/ do
|
|
918
|
+
groups Text::Whitespace, Comment::Preproc, Text::Whitespace, Name::Namespace
|
|
919
|
+
end
|
|
920
|
+
rule %r/^(\p{Space}*)(=[A-Za-z][#{w}]*)/ do
|
|
921
|
+
groups Text::Whitespace, Comment::Preproc
|
|
922
|
+
end
|
|
923
|
+
# formatting codes: B<bold>, I<italic>, C<code>, L<link>, ...
|
|
924
|
+
rule %r/(B)(<)([^<>\n]*)(>)/ do
|
|
925
|
+
groups Name::Decorator, Punctuation, Generic::Strong, Punctuation
|
|
926
|
+
end
|
|
927
|
+
rule %r/(I)(<)([^<>\n]*)(>)/ do
|
|
928
|
+
groups Name::Decorator, Punctuation, Generic::Emph, Punctuation
|
|
929
|
+
end
|
|
930
|
+
rule %r/([CKTV])(<)([^<>\n]*)(>)/ do
|
|
931
|
+
groups Name::Decorator, Punctuation, Str::Backtick, Punctuation
|
|
932
|
+
end
|
|
933
|
+
rule %r/([ALENPRSUXZ])(<)([^<>\n]*)(>)/ do
|
|
934
|
+
groups Name::Decorator, Punctuation, Comment::Multiline, Punctuation
|
|
935
|
+
end
|
|
936
|
+
rule %r/[^\n]+?(?=[A-Z]<|$)/, Comment::Multiline
|
|
937
|
+
rule %r/[A-Z]/, Comment::Multiline
|
|
938
|
+
rule %r/\n/, Text::Whitespace
|
|
939
|
+
end
|
|
940
|
+
|
|
941
|
+
# what can appear inside an interpolating string
|
|
942
|
+
state :interpolation do
|
|
943
|
+
rule %r/\\(?:[abefnrt0"'\\$@%&{}<>«»]|[xXoOcCdD]\[[^\]\n]*\]|x[0-9a-fA-F]+)/, Str::Escape
|
|
944
|
+
rule %r/\\./m, Str::Double
|
|
945
|
+
# What interpolates is code, and is lexed as code, so that a
|
|
946
|
+
# variable looks the same inside a string as outside of it.
|
|
947
|
+
# - contextualizers: $(...), @(...)
|
|
948
|
+
# - scalars: $x
|
|
949
|
+
# - arrays and hashes only if something in brackets or parentheses
|
|
950
|
+
# follows: @a[0], %h<a>, @a.elems()
|
|
951
|
+
# - functions only if they are called: &f()
|
|
952
|
+
# Each can be followed by subscripts and by method calls, which
|
|
953
|
+
# need their parentheses here: $x[0]<a>, $x.uc(), @a.sort().join(',')
|
|
954
|
+
rule %r/(?:[$@%&]#{arguments}#{postfix}*|\$(?:[*.!^?=~:]?#{qualified_ident}|[!\/&¢]|\p{Nd}+|<[^>\n]+>)#{postfix}*|[@%][*.!^?=~:]?#{qualified_ident}#{postfix}+|&#{ident}#{arguments}#{postfix}*)/ do |m|
|
|
955
|
+
sublex m[0]
|
|
956
|
+
end
|
|
957
|
+
rule %r/\{/ do
|
|
958
|
+
token Punctuation
|
|
959
|
+
@brace_levels << 1
|
|
960
|
+
push :embedded
|
|
961
|
+
end
|
|
962
|
+
end
|
|
963
|
+
|
|
964
|
+
# the body of a qq string or of an interpolating heredoc
|
|
965
|
+
state :interpolated do
|
|
966
|
+
mixin :interpolation
|
|
967
|
+
rule %r/[^\\$@%&{]+/, Str::Double
|
|
968
|
+
rule %r/[$@%&\\]/, Str::Double
|
|
969
|
+
end
|
|
970
|
+
|
|
971
|
+
state :dq_string do
|
|
972
|
+
rule %r/"/, Str::Double, :pop!
|
|
973
|
+
mixin :interpolation
|
|
974
|
+
rule %r/[^"\\$@%&{]+/, Str::Double
|
|
975
|
+
rule %r/[$@%&\\]/, Str::Double
|
|
976
|
+
end
|
|
977
|
+
|
|
978
|
+
state :dq_curly do
|
|
979
|
+
rule %r/[”“]/, Str::Double, :pop!
|
|
980
|
+
mixin :interpolation
|
|
981
|
+
rule %r/[^”“\\$@%&{]+/, Str::Double
|
|
982
|
+
rule %r/[$@%&\\]/, Str::Double
|
|
983
|
+
end
|
|
984
|
+
|
|
985
|
+
state :ww_guillemets do
|
|
986
|
+
rule %r/»/, Str::Double, :pop!
|
|
987
|
+
mixin :interpolation
|
|
988
|
+
rule %r/[^»\\$@%&{]+/, Str::Double
|
|
989
|
+
rule %r/[$@%&\\]/, Str::Double
|
|
990
|
+
end
|
|
991
|
+
|
|
992
|
+
state :ww_angles do
|
|
993
|
+
rule %r/>>/, Str::Double, :pop!
|
|
994
|
+
mixin :interpolation
|
|
995
|
+
rule %r/[^>\\$@%&{]+/, Str::Double
|
|
996
|
+
# (a lone '>' does not close the quote)
|
|
997
|
+
rule %r/[$@%&\\>]/, Str::Double
|
|
998
|
+
end
|
|
999
|
+
end
|
|
1000
|
+
end
|
|
1001
|
+
end
|
data/lib/rouge/raku.rb
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'rouge'
|
|
4
|
+
|
|
5
|
+
require_relative 'raku/version'
|
|
6
|
+
require_relative 'lexers/raku'
|
|
7
|
+
|
|
8
|
+
module Rouge
|
|
9
|
+
module Lexers
|
|
10
|
+
class Raku
|
|
11
|
+
# Rouge looks for the demo of a lexer among its own files. This one
|
|
12
|
+
# ships with this gem.
|
|
13
|
+
def self.demo_file(*)
|
|
14
|
+
Pathname.new(File.join(__dir__, 'demos', 'raku'))
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# The lexer looks behind the text it is matching ("not after a dot"),
|
|
18
|
+
# which only works if the scanner anchors on the whole input and not
|
|
19
|
+
# on the place it has reached. Rouge sets its scanner up that way from
|
|
20
|
+
# version 5.0 on. Do the same for the versions before it.
|
|
21
|
+
if Gem::Version.new(Rouge.version) < Gem::Version.new('5.0')
|
|
22
|
+
def stream_tokens(str, &b)
|
|
23
|
+
stream = StringScanner.new(str, fixed_anchor: true)
|
|
24
|
+
|
|
25
|
+
@current_stream = stream
|
|
26
|
+
@output_stream = b
|
|
27
|
+
@states = self.class.states
|
|
28
|
+
@null_steps = 0
|
|
29
|
+
|
|
30
|
+
until stream.eos?
|
|
31
|
+
b.call(Token::Tokens::Error, stream.getch) unless step(state, stream)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
data/lib/rouge-raku.rb
ADDED
data/rouge-raku.gemspec
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require './lib/rouge/raku/version'
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |s|
|
|
6
|
+
s.name = "rouge-raku"
|
|
7
|
+
s.version = Rouge::Raku::VERSION
|
|
8
|
+
s.authors = ["Naoum Hankache"]
|
|
9
|
+
s.email = ["naoum88@gmail.com"]
|
|
10
|
+
s.summary = "A Raku lexer for the Rouge syntax highlighter"
|
|
11
|
+
s.description = <<-DESC.strip.gsub(/\s+/, ' ')
|
|
12
|
+
Adds syntax highlighting for the Raku programming language
|
|
13
|
+
to Rouge, as a plugin gem.
|
|
14
|
+
DESC
|
|
15
|
+
s.homepage = "https://github.com/hankache/rouge-raku"
|
|
16
|
+
s.files = Dir['CHANGELOG.md', 'Gemfile', 'LICENSE', 'README.md', 'rouge-raku.gemspec', 'lib/**/*.rb', 'lib/rouge/demos/*']
|
|
17
|
+
s.licenses = ['MIT']
|
|
18
|
+
s.required_ruby_version = '>= 3.0'
|
|
19
|
+
s.metadata = {
|
|
20
|
+
"bug_tracker_uri" => "https://github.com/hankache/rouge-raku/issues",
|
|
21
|
+
"changelog_uri" => "https://github.com/hankache/rouge-raku/blob/main/CHANGELOG.md",
|
|
22
|
+
"source_code_uri" => "https://github.com/hankache/rouge-raku",
|
|
23
|
+
"rubygems_mfa_required" => "true"
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
s.add_dependency 'rouge', '>= 3.26', '< 6'
|
|
27
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: rouge-raku
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Naoum Hankache
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-10-05 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: rouge
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '3.26'
|
|
20
|
+
- - "<"
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: '6'
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - ">="
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '3.26'
|
|
30
|
+
- - "<"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '6'
|
|
33
|
+
description: Adds syntax highlighting for the Raku programming language to Rouge,
|
|
34
|
+
as a plugin gem.
|
|
35
|
+
email:
|
|
36
|
+
- naoum88@gmail.com
|
|
37
|
+
executables: []
|
|
38
|
+
extensions: []
|
|
39
|
+
extra_rdoc_files: []
|
|
40
|
+
files:
|
|
41
|
+
- CHANGELOG.md
|
|
42
|
+
- Gemfile
|
|
43
|
+
- LICENSE
|
|
44
|
+
- README.md
|
|
45
|
+
- lib/rouge-raku.rb
|
|
46
|
+
- lib/rouge/demos/raku
|
|
47
|
+
- lib/rouge/lexers/raku.rb
|
|
48
|
+
- lib/rouge/raku.rb
|
|
49
|
+
- lib/rouge/raku/version.rb
|
|
50
|
+
- rouge-raku.gemspec
|
|
51
|
+
homepage: https://github.com/hankache/rouge-raku
|
|
52
|
+
licenses:
|
|
53
|
+
- MIT
|
|
54
|
+
metadata:
|
|
55
|
+
bug_tracker_uri: https://github.com/hankache/rouge-raku/issues
|
|
56
|
+
changelog_uri: https://github.com/hankache/rouge-raku/blob/main/CHANGELOG.md
|
|
57
|
+
source_code_uri: https://github.com/hankache/rouge-raku
|
|
58
|
+
rubygems_mfa_required: 'true'
|
|
59
|
+
post_install_message:
|
|
60
|
+
rdoc_options: []
|
|
61
|
+
require_paths:
|
|
62
|
+
- lib
|
|
63
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '3.0'
|
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
69
|
+
requirements:
|
|
70
|
+
- - ">="
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: '0'
|
|
73
|
+
requirements: []
|
|
74
|
+
rubygems_version: 3.4.20
|
|
75
|
+
signing_key:
|
|
76
|
+
specification_version: 4
|
|
77
|
+
summary: A Raku lexer for the Rouge syntax highlighter
|
|
78
|
+
test_files: []
|