pygments.rb 0.2.8 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,330 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ pygments.lexers.github
4
+ ~~~~~~~~~~~~~~~~~~~
5
+
6
+ Custom lexers for GitHub.com
7
+
8
+ :copyright: Copyright 2012 by GitHub, Inc
9
+ :license: BSD, see LICENSE for details.
10
+ """
11
+ import re
12
+
13
+ from pygments.lexer import RegexLexer, include, bygroups, using, DelegatingLexer
14
+ from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
15
+ Other, Keyword, Operator, Literal
16
+
17
+ __all__ = ['Dasm16Lexer', 'PuppetLexer', 'AugeasLexer']
18
+
19
+ class Dasm16Lexer(RegexLexer):
20
+ """
21
+ Simple lexer for DCPU-16 Assembly
22
+
23
+ Check http://0x10c.com/doc/dcpu-16.txt
24
+ """
25
+ name = 'dasm16'
26
+ aliases = ['DASM16']
27
+ filenames = ['*.dasm16', '*.dasm']
28
+ mimetypes = ['text/x-dasm16']
29
+
30
+ INSTRUCTIONS = [
31
+ 'SET', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'SHL',
32
+ 'SHR', 'AND', 'BOR', 'XOR', 'IFE', 'IFN', 'IFG'
33
+ 'IFB', 'JSR'
34
+ ]
35
+
36
+ REGISTERS = [
37
+ 'A', 'B', 'C',
38
+ 'X', 'Y', 'Z',
39
+ 'I', 'J',
40
+ 'SP', 'PC',
41
+ 'POP', 'PEEK', 'PUSH'
42
+ ]
43
+
44
+ # Regexes yo
45
+ char = r'[a-zA-Z$._0-9@]'
46
+ identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
47
+ number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
48
+ instruction = r'(?i)(' + '|'.join(INSTRUCTIONS) + ')'
49
+
50
+ def guess_identifier(lexer, match):
51
+ ident = match.group(0)
52
+ klass = Name.Variable if ident in lexer.REGISTERS else Name.Label
53
+ yield match.start(), klass, ident
54
+
55
+ tokens = {
56
+ 'root': [
57
+ include('whitespace'),
58
+ (':' + identifier, Name.Label),
59
+ (instruction, Name.Function, 'instruction-args'),
60
+ (r'[\r\n]+', Text)
61
+ ],
62
+
63
+ 'arg' : [
64
+ (identifier, guess_identifier),
65
+ (number, Number.Integer),
66
+ ],
67
+
68
+ 'deref' : [
69
+ (r'\+', Punctuation),
70
+ (r'\]', Punctuation, '#pop'),
71
+ include('arg'),
72
+ include('whitespace')
73
+ ],
74
+
75
+ 'instruction-args': [
76
+ (r',', Punctuation),
77
+ (r'\[', Punctuation, 'deref'),
78
+ include('arg'),
79
+
80
+ (r'[\r\n]+', Text, '#pop'),
81
+ (r';.*?$', Comment, '#pop'),
82
+ include('whitespace')
83
+ ],
84
+ 'whitespace': [
85
+ (r'\n', Text),
86
+ (r'\s+', Text),
87
+ (r';.*?\n', Comment)
88
+ ],
89
+ }
90
+
91
+ class PuppetLexer(RegexLexer):
92
+ name = 'Puppet'
93
+ aliases = ['puppet']
94
+ filenames = ['*.pp']
95
+
96
+ tokens = {
97
+ 'root': [
98
+ include('puppet'),
99
+ ],
100
+ 'puppet': [
101
+ include('comments'),
102
+ (r'(class|define)', Keyword.Declaration, ('block','class_name')),
103
+ (r'node', Keyword.Declaration, ('block', 'node_name')),
104
+ (r'elsif', Keyword.Reserved, ('block', 'conditional')),
105
+ (r'if', Keyword.Reserved, ('block', 'conditional')),
106
+ (r'unless', Keyword.Reserved, ('block', 'conditional')),
107
+ (r'(else)(\s*)(\{)', bygroups(Keyword.Reserved, Text, Punctuation), 'block'),
108
+ (r'case', Keyword.Reserved, ('case', 'conditional')),
109
+ (r'(::)?([A-Z][\w:]+)+(\s*)(<{1,2}\|)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'spaceinvader'),
110
+ (r'(::)?([A-Z][\w:]+)+(\s*)(\{)', bygroups(Name.Class, Name.Class, Text, Punctuation), 'type'),
111
+ (r'(::)?([A-Z][\w:]+)+(\s*)(\[)', bygroups(Name.Class, Name.Class, Text, Punctuation), ('type', 'override_name')),
112
+ (r'(@{0,2}[\w:]+)(\s*)(\{)(\s*)', bygroups(Name.Class, Text, Punctuation, Text), ('type', 'namevar')),
113
+ (r'\$(::)?(\w+::)*\w+', Name.Variable, 'var_assign'),
114
+ (r'include', Keyword.Namespace, 'include'),
115
+ (r'import', Keyword.Namespace, 'import'),
116
+ (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
117
+ (r'\s', Text),
118
+ ],
119
+ 'block': [
120
+ include('puppet'),
121
+ (r'\}', Text, '#pop'),
122
+ ],
123
+ 'override_name': [
124
+ include('strings'),
125
+ include('variables'),
126
+ (r'\]', Punctuation),
127
+ (r'\s', Text),
128
+ (r'\{', Punctuation, '#pop'),
129
+ ],
130
+ 'node_name': [
131
+ (r'inherits', Keyword.Declaration),
132
+ (r'[\w\.]+', String),
133
+ include('strings'),
134
+ include('variables'),
135
+ (r',', Punctuation),
136
+ (r'\s', Text),
137
+ (r'\{', Punctuation, '#pop'),
138
+ ],
139
+ 'class_name': [
140
+ (r'inherits', Keyword.Declaration),
141
+ (r'[\w:]+', Name.Class),
142
+ (r'\s', Text),
143
+ (r'\{', Punctuation, '#pop'),
144
+ (r'\(', Punctuation, 'paramlist'),
145
+ ],
146
+ 'include': [
147
+ (r'\n', Text, '#pop'),
148
+ (r'[\w:]+', Name.Class),
149
+ include('value'),
150
+ (r'\s', Text),
151
+ ],
152
+ 'import': [
153
+ (r'\n', Text, '#pop'),
154
+ (r'[\/\w\.]+', String),
155
+ include('value'),
156
+ (r'\s', Text),
157
+ ],
158
+ 'case': [
159
+ (r'(default)(:)(\s*)(\{)', bygroups(Keyword.Reserved, Punctuation, Text, Punctuation), 'block'),
160
+ include('case_values'),
161
+ (r'(:)(\s*)(\{)', bygroups(Punctuation, Text, Punctuation), 'block'),
162
+ (r'\s', Text),
163
+ (r'\}', Punctuation, '#pop'),
164
+ ],
165
+ 'case_values': [
166
+ include('value'),
167
+ (r',', Punctuation),
168
+ ],
169
+ 'comments': [
170
+ (r'\s*#.*\n', Comment.Singleline),
171
+ ],
172
+ 'strings': [
173
+ (r"'.*?'", String.Single),
174
+ (r'\w+', String.Symbol),
175
+ (r'"', String.Double, 'dblstring'),
176
+ (r'\/.+?\/', String.Regex),
177
+ ],
178
+ 'dblstring': [
179
+ (r'\$\{.+?\}', String.Interpol),
180
+ (r'(?:\\(?:[bdefnrstv\'"\$\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))', String.Escape),
181
+ (r'[^"\\\$]+', String.Double),
182
+ (r'\$', String.Double),
183
+ (r'"', String.Double, '#pop'),
184
+ ],
185
+ 'variables': [
186
+ (r'\$(::)?(\w+::)*\w+', Name.Variable),
187
+ ],
188
+ 'var_assign': [
189
+ (r'\[', Punctuation, ('#pop', 'array')),
190
+ (r'\{', Punctuation, ('#pop', 'hash')),
191
+ (r'(\s*)(=)(\s*)', bygroups(Text, Operator, Text)),
192
+ (r'(\(|\))', Punctuation),
193
+ include('operators'),
194
+ include('value'),
195
+ (r'\s', Text, '#pop'),
196
+ ],
197
+ 'booleans': [
198
+ (r'(true|false)', Literal),
199
+ ],
200
+ 'operators': [
201
+ (r'(\s*)(==|=~|\*|-|\+|<<|>>|!=|!~|!|>=|<=|<|>|and|or|in)(\s*)', bygroups(Text, Operator, Text)),
202
+ ],
203
+ 'conditional': [
204
+ include('operators'),
205
+ include('strings'),
206
+ include('variables'),
207
+ (r'\[', Punctuation, 'array'),
208
+ (r'\(', Punctuation, 'conditional'),
209
+ (r'\{', Punctuation, '#pop'),
210
+ (r'\)', Punctuation, '#pop'),
211
+ (r'\s', Text),
212
+ ],
213
+ 'spaceinvader': [
214
+ include('operators'),
215
+ include('strings'),
216
+ include('variables'),
217
+ (r'\[', Punctuation, 'array'),
218
+ (r'\(', Punctuation, 'conditional'),
219
+ (r'\s', Text),
220
+ (r'\|>{1,2}', Punctuation, '#pop'),
221
+ ],
222
+ 'namevar': [
223
+ include('value'),
224
+ (r'\[', Punctuation, 'array'),
225
+ (r'\s', Text),
226
+ (r':', Punctuation, '#pop'),
227
+ (r'\}', Punctuation, '#pop'),
228
+ ],
229
+ 'function': [
230
+ (r'\[', Punctuation, 'array'),
231
+ include('value'),
232
+ (r',', Punctuation),
233
+ (r'\s', Text),
234
+ (r'\)', Punctuation, '#pop'),
235
+ ],
236
+ 'paramlist': [
237
+ include('value'),
238
+ (r'=', Punctuation),
239
+ (r',', Punctuation),
240
+ (r'\s', Text),
241
+ (r'\[', Punctuation, 'array'),
242
+ (r'\)', Punctuation, '#pop'),
243
+ ],
244
+ 'type': [
245
+ (r'(\w+)(\s*)(=>)(\s*)', bygroups(Name.Tag, Text, Punctuation, Text), 'param_value'),
246
+ (r'\}', Punctuation, '#pop'),
247
+ (r'\s', Text),
248
+ include('comments'),
249
+ (r'', Text, 'namevar'),
250
+ ],
251
+ 'value': [
252
+ (r'[\d\.]', Number),
253
+ (r'([A-Z][\w:]+)+(\[)', bygroups(Name.Class, Punctuation), 'array'),
254
+ (r'(\w+)(\()', bygroups(Name.Function, Punctuation), 'function'),
255
+ include('strings'),
256
+ include('variables'),
257
+ include('comments'),
258
+ include('booleans'),
259
+ (r'(\s*)(\?)(\s*)(\{)', bygroups(Text, Punctuation, Text, Punctuation), 'selector'),
260
+ (r'\{', Punctuation, 'hash'),
261
+ ],
262
+ 'selector': [
263
+ (r'default', Keyword.Reserved),
264
+ include('value'),
265
+ (r'=>', Punctuation),
266
+ (r',', Punctuation),
267
+ (r'\s', Text),
268
+ (r'\}', Punctuation, '#pop'),
269
+ ],
270
+ 'param_value': [
271
+ include('value'),
272
+ (r'\[', Punctuation, 'array'),
273
+ (r',', Punctuation, '#pop'),
274
+ (r';', Punctuation, '#pop'),
275
+ (r'\s', Text, '#pop'),
276
+ (r'', Text, '#pop'),
277
+ ],
278
+ 'array': [
279
+ include('value'),
280
+ (r'\[', Punctuation, 'array'),
281
+ (r',', Punctuation),
282
+ (r'\s', Text),
283
+ (r'\]', Punctuation, '#pop'),
284
+ ],
285
+ 'hash': [
286
+ include('value'),
287
+ (r'\s', Text),
288
+ (r'=>', Punctuation),
289
+ (r',', Punctuation),
290
+ (r'\}', Punctuation, '#pop'),
291
+ ],
292
+ }
293
+
294
+ class AugeasLexer(RegexLexer):
295
+ name = 'Augeas'
296
+ aliases = ['augeas']
297
+ filenames = ['*.aug']
298
+
299
+ tokens = {
300
+ 'root': [
301
+ (r'(module)(\s*)([^\s=]+)', bygroups(Keyword.Namespace, Text, Name.Namespace)),
302
+ (r'(let)(\s*)([^\s=]+)', bygroups(Keyword.Declaration, Text, Name.Variable)),
303
+ (r'(del|store|value|counter|seq|key|label|autoload|incl|excl|transform|test|get|put)(\s+)', bygroups(Name.Builtin, Text)),
304
+ (r'(\()([^\:]+)(\:)(unit|string|regexp|lens|tree|filter)(\))', bygroups(Punctuation, Name.Variable, Punctuation, Keyword.Type, Punctuation)),
305
+ (r'\(\*', Comment.Multiline, 'comment'),
306
+ (r'[\+=\|\.\*\;\?-]', Operator),
307
+ (r'[\[\]\(\)\{\}]', Operator),
308
+ (r'"', String.Double, 'string'),
309
+ (r'\/', String.Regex, 'regex'),
310
+ (r'([A-Z]\w*)(\.)(\w+)', bygroups(Name.Namespace, Punctuation, Name.Variable)),
311
+ (r'.', Name.Variable),
312
+ (r'\s', Text),
313
+ ],
314
+ 'string': [
315
+ (r'\\.', String.Escape),
316
+ (r'[^"]', String.Double),
317
+ (r'"', String.Double, '#pop'),
318
+ ],
319
+ 'regex': [
320
+ (r'\\.', String.Escape),
321
+ (r'[^\/]', String.Regex),
322
+ (r'\/', String.Regex, '#pop'),
323
+ ],
324
+ 'comment': [
325
+ (r'[^*\)]', Comment.Multiline),
326
+ (r'\(\*', Comment.Multiline, '#push'),
327
+ (r'\*\)', Comment.Multiline, '#pop'),
328
+ (r'[\*\)]', Comment.Multiline)
329
+ ],
330
+ }
@@ -11,15 +11,170 @@
11
11
 
12
12
  import re
13
13
 
14
- from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions
14
+ from pygments.lexer import Lexer, RegexLexer, bygroups, include, \
15
+ combined, do_insertions
15
16
  from pygments.token import Comment, String, Punctuation, Keyword, Name, \
16
17
  Operator, Number, Text, Generic
17
18
 
18
19
  from pygments.lexers.agile import PythonLexer
19
20
  from pygments.lexers import _scilab_builtins
20
21
 
21
- __all__ = ['MuPADLexer', 'MatlabLexer', 'MatlabSessionLexer', 'OctaveLexer',
22
- 'ScilabLexer', 'NumPyLexer', 'RConsoleLexer', 'SLexer']
22
+ __all__ = ['JuliaLexer', 'JuliaConsoleLexer', 'MuPADLexer', 'MatlabLexer',
23
+ 'MatlabSessionLexer', 'OctaveLexer', 'ScilabLexer', 'NumPyLexer',
24
+ 'RConsoleLexer', 'SLexer']
25
+
26
+
27
+ class JuliaLexer(RegexLexer):
28
+ name = 'Julia'
29
+ aliases = ['julia','jl']
30
+ filenames = ['*.jl']
31
+ mimetypes = ['text/x-julia','application/x-julia']
32
+
33
+ builtins = [
34
+ 'exit','whos','edit','load','is','isa','isequal','typeof','tuple',
35
+ 'ntuple','uid','hash','finalizer','convert','promote','subtype',
36
+ 'typemin','typemax','realmin','realmax','sizeof','eps','promote_type',
37
+ 'method_exists','applicable','invoke','dlopen','dlsym','system',
38
+ 'error','throw','assert','new','Inf','Nan','pi','im',
39
+ ]
40
+
41
+ tokens = {
42
+ 'root': [
43
+ (r'\n', Text),
44
+ (r'[^\S\n]+', Text),
45
+ (r'#.*$', Comment),
46
+ (r'[]{}:(),;[@]', Punctuation),
47
+ (r'\\\n', Text),
48
+ (r'\\', Text),
49
+
50
+ # keywords
51
+ (r'(begin|while|for|in|return|break|continue|'
52
+ r'macro|quote|let|if|elseif|else|try|catch|end|'
53
+ r'bitstype|ccall)\b', Keyword),
54
+ (r'(local|global|const)\b', Keyword.Declaration),
55
+ (r'(module|import|export)\b', Keyword.Reserved),
56
+ (r'(Bool|Int|Int8|Int16|Int32|Int64|Uint|Uint8|Uint16|Uint32|Uint64'
57
+ r'|Float32|Float64|Complex64|Complex128|Any|Nothing|None)\b',
58
+ Keyword.Type),
59
+
60
+ # functions
61
+ (r'(function)((?:\s|\\\s)+)',
62
+ bygroups(Keyword,Name.Function), 'funcname'),
63
+
64
+ # types
65
+ (r'(type|typealias|abstract)((?:\s|\\\s)+)',
66
+ bygroups(Keyword,Name.Class), 'typename'),
67
+
68
+ # operators
69
+ (r'==|!=|<=|>=|->|&&|\|\||::|<:|[-~+/*%=<>&^|.?!$]', Operator),
70
+ (r'\.\*|\.\^|\.\\|\.\/|\\', Operator),
71
+
72
+ # builtins
73
+ ('(' + '|'.join(builtins) + r')\b', Name.Builtin),
74
+
75
+ # backticks
76
+ (r'`(?s).*?`', String.Backtick),
77
+
78
+ # chars
79
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),
80
+
81
+ # try to match trailing transpose
82
+ (r'(?<=[.\w\)\]])\'', Operator),
83
+
84
+ # strings
85
+ (r'(?:[IL])"', String, 'string'),
86
+ (r'[E]?"', String, combined('stringescape', 'string')),
87
+
88
+ # names
89
+ (r'@[a-zA-Z0-9_.]+', Name.Decorator),
90
+ (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
91
+
92
+ # numbers
93
+ (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
94
+ (r'\d+[eE][+-]?[0-9]+', Number.Float),
95
+ (r'0[0-7]+', Number.Oct),
96
+ (r'0[xX][a-fA-F0-9]+', Number.Hex),
97
+ (r'\d+', Number.Integer)
98
+ ],
99
+
100
+ 'funcname': [
101
+ ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop'),
102
+ ('\([^\s\w{]{1,2}\)', Operator, '#pop'),
103
+ ('[^\s\w{]{1,2}', Operator, '#pop'),
104
+ ],
105
+
106
+ 'typename': [
107
+ ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop')
108
+ ],
109
+
110
+ 'stringescape': [
111
+ (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
112
+ r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
113
+ ],
114
+
115
+ 'string': [
116
+ (r'"', String, '#pop'),
117
+ (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
118
+ (r'\$(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?',
119
+ String.Interpol),
120
+ (r'[^\\"$]+', String),
121
+ # quotes, dollar signs, and backslashes must be parsed one at a time
122
+ (r'["\\]', String),
123
+ # unhandled string formatting sign
124
+ (r'\$', String)
125
+ ],
126
+ }
127
+
128
+ def analyse_text(text):
129
+ return shebang_matches(text, r'julia')
130
+
131
+
132
+ line_re = re.compile('.*?\n')
133
+
134
+ class JuliaConsoleLexer(Lexer):
135
+ """
136
+ For Julia console sessions. Modeled after MatlabSessionLexer.
137
+ """
138
+ name = 'Julia console'
139
+ aliases = ['jlcon']
140
+
141
+ def get_tokens_unprocessed(self, text):
142
+ jllexer = JuliaLexer(**self.options)
143
+
144
+ curcode = ''
145
+ insertions = []
146
+
147
+ for match in line_re.finditer(text):
148
+ line = match.group()
149
+
150
+ if line.startswith('julia>'):
151
+ insertions.append((len(curcode),
152
+ [(0, Generic.Prompt, line[:3])]))
153
+ curcode += line[3:]
154
+
155
+ elif line.startswith(' '):
156
+
157
+ idx = len(curcode)
158
+
159
+ # without is showing error on same line as before...?
160
+ line = "\n" + line
161
+ token = (0, Generic.Traceback, line)
162
+ insertions.append((idx, [token]))
163
+
164
+ else:
165
+ if curcode:
166
+ for item in do_insertions(
167
+ insertions, jllexer.get_tokens_unprocessed(curcode)):
168
+ yield item
169
+ curcode = ''
170
+ insertions = []
171
+
172
+ yield match.start(), Generic.Output, line
173
+
174
+ if curcode: # or item:
175
+ for item in do_insertions(
176
+ insertions, jllexer.get_tokens_unprocessed(curcode)):
177
+ yield item
23
178
 
24
179
 
25
180
  class MuPADLexer(RegexLexer):