rouge-lexer-spl 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95bcba36175336b5f4bdfb541516536df46990fb43bf2530690815c758ee96bf
-  data.tar.gz: 55818398ba0266efba91869c7d5f508724421d5165c90a0cce98c5e285b812e2
+  metadata.gz: 450ea9a0cce00914039ec808a58477b6baadde9b429946d4ba68ed0121e980f4
+  data.tar.gz: 9119b5cd29f2034885699c1ce7589d2f4a0a3cf6acce2e6150b6d1fac12b8fd0
 SHA512:
-  metadata.gz: ef14e53f0f449fa595f51a3227c2fdef71fbd08875f056b57827a01da3f2f4ecd7b50182304fddd948ce8d19376a915e707657399f78f1f43b58bc319e8e4c45
-  data.tar.gz: cf377703f943a0dc81a2c4802772f82aaa382e9830db8deae252dc16b70fb93e770f954e4f6070013a5be3ff05997caab445fa73f12251d0b351cb455ae14cb5
+  metadata.gz: ddf2519c5882dcf07e3edd0ed8ab29eaefd8cf2074ee5709825357a2802d24b8362f9249a45fa6a9832675361518472215b3b3671cdab6933ec930481d3a3fcd
+  data.tar.gz: d6ddf3c9803e7d747693965702eab614eb2f396c6681026a87bf55e1f5005d7b18e772a8a58d4212a33bf8dba5164438ac98411cbc8cb00a722290d727303d3b

data/README.md

@@ -0,0 +1,94 @@
+# rouge-lexer-spl
+
+A Rouge lexer plugin for Splunk's Search Processing Language (SPL). Rouge is the default syntax highlighter for Jekyll (and therefore GitHub Pages). This gem adds SPL support to Rouge.
+
+## Installation
+
+Install the gem directly:
+
+```sh
+gem install rouge-lexer-spl
+```
+
+Or add it to your `Gemfile`:
+
+```ruby
+gem 'rouge-lexer-spl'
+```
+
+Then run:
+
+```sh
+bundle install
+```
+
+## Usage
+
+Once installed, Rouge will automatically discover the lexer. You can use `spl`, `splunk`, or `splunk-spl` as the language tag in fenced code blocks:
+
+````markdown
+```spl
+index=web sourcetype=access_combined | stats count by status
+```
+````
+
+## Jekyll / GitHub Pages
+
+Add the gem to your site's `Gemfile` inside the `:jekyll_plugins` group:
+
+```ruby
+group :jekyll_plugins do
+  gem "rouge-lexer-spl"
+end
+```
+
+Run `bundle install`, then use ` ```spl ` fences in your posts and pages. Jekyll will pick up the lexer automatically via Rouge's plugin discovery.
+
+## Development
+
+Install dependencies:
+
+```sh
+bundle install
+```
+
+Run the test suite:
+
+```sh
+bundle exec rake
+```
+
+Start the visual preview server (available at http://localhost:9292):
+
+```sh
+bundle exec rake server
+```
+
+Run the terminal preview script:
+
+```sh
+ruby preview.rb
+```
+
+Enable debug mode to print each token and its value:
+
+```sh
+DEBUG=1 ruby preview.rb
+```
+
+### Iterative testing workflow
+
+1. Run `bundle exec rake` to check for test failures and error tokens.
+2. Start the server with `bundle exec rake server`.
+3. In another terminal, check for error tokens in the rendered output:
+
+   ```sh
+   curl -s http://localhost:9292 | grep 'class="err"'
+   ```
+
+4. Fix any error tokens in `lib/rouge/lexers/spl.rb`.
+5. Repeat until no error tokens remain.
+
+## License
+
+MIT

data/lib/rouge/lexers/spl.rb

@@ -16,87 +16,117 @@ module Rouge
         return true if text =~ /\bindex\s*=\s*\w+/i && text =~ /\bsourcetype\s*=\s*/i
       end
 
-      # SPL commands (from Splunk command quick reference)
+      # SPL commands
+      # Sources: https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/quick-reference/command-quick-reference
+      #          https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/quick-reference/command-types
       def self.commands
         @commands ||= Set.new %w(
           abstract accum addcoltotals addinfo addtotals analyzefields
           anomalies anomalousvalue anomalydetection append appendcols
-          appendpipe arules associate autoregress bin bucket bucketdir
-          chart cluster cofilter collect concurrency contingency convert
-          correlate datamodel dbinspect dedup delete delta diff erex eval
-          eventcount eventstats extract filldown fillnull findtypes
-          folderize foreach format from gauge gentimes geom geomfilter
-          geostats head highlight history iconify inputcsv inputlookup
-          iplocation join kmeans kvform loadjob localize localop lookup
-          makecontinuous makemv makeresults map mcollect metadata
-          metasearch meventcollect mpreview msearch mstats multikv
-          multisearch mvcombine mvexpand nomv outlier outputcsv
-          outputlookup outputtext overlap pivot predict rangemap rare
-          redistribute regex reltime rename replace require rest return
-          reverse rex rtorder savedsearch script run scrub search
-          searchtxn selfjoin sendalert sendemail set setfields sichart
-          sirare sistats sitimechart sitop sort spath stats strcat
+          appendpipe arules associate autoregress awssnsalert bin bucket
+          bucketdir chart cluster cofilter collect concurrency contingency
+          convert correlate ctable datamodel datamodelsimple dbinspect
+          dbxquery dedup delete delta diff entitymerge erex eval eventcount
+          eventstats extract fieldformat fields fieldsummary filldown
+          fillnull findtypes folderize foreach format from fromjson gauge
+          gentimes geom geomfilter geostats head highlight history iconify
+          ingestpreview inputcsv inputintelligence inputlookup iplocation
+          join kmeans kvform loadjob localize localop lookup makecontinuous
+          makemv makeresults map mcollect metadata metasearch meventcollect
+          mpreview msearch mstats multikv multisearch mvcombine mvexpand
+          nomv noop outlier outputcsv outputlookup outputtext overlap pivot
+          predict rangemap rare redistribute regex reltime rename replace
+          require rest return reverse rex rtorder run savedsearch script
+          scrub search searchtxn selfjoin sendalert sendemail set setfields
+          sichart sirare sistats sitimechart sitop snowevent snoweventstream
+          snowincident snowincidentstream sort spath stats strcat
           streamstats table tags tail timechart timewrap tojson top
           transaction transpose trendline tscollect tstats typeahead
           typelearner typer union uniq untable walklex where x11 xmlkv
-          xmlunescape xpath xyseries kv
+          xmlunescape xpath xyseries
         )
       end
 
-      # Evaluation functions (from Splunk evaluation functions reference)
+      # Evaluation functions
+      # Sources:
+      #   Bitwise:        https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/bitwise-functions
+      #   Comparison:     https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/comparison-and-conditional-functions
+      #   Conversion:     https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/conversion-functions
+      #   Cryptographic:  https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/cryptographic-functions
+      #   Date/Time:      https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/date-and-time-functions
+      #   Informational:  https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/informational-functions
+      #   JSON:           https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/json-functions
+      #   Mathematical:   https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/mathematical-functions
+      #   Multivalue:     https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/multivalue-eval-functions
+      #   Statistical:    https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/statistical-eval-functions
+      #   Text:           https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/text-functions
+      #   Trig:           https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/evaluation-functions/trig-and-hyperbolic-functions
+      #   Convert cmd:    https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/search-commands/convert
       def self.eval_functions
         @eval_functions ||= Set.new %w(
           abs acos acosh asin asinh atan atan2 atanh avg
-          bit_and bit_or bit_not bit_xor bit_shift_left bit_shift_right
-          case cidrmatch ceiling coalesce commands cos cosh exact exp
-          false floor hypot if in ipmask isarray isbool isdouble isint
-          ismv isnotnull isnull isnum isobject isstr json json_append
-          json_array json_array_to_mv json_delete json_entries
+          bit_and bit_not bit_or bit_shift_left bit_shift_right bit_xor
+          case ceil ceiling cidrmatch coalesce commands cos cosh
+          ctime dur2sec
+          exact exp false floor hypot if ipmask
+          isarray isbool isdouble isint ismv isnotnull isnull isnum isobject isstr
+          json json_append json_array json_array_to_mv json_delete json_entries
           json_extend json_extract json_extract_exact json_has_key_exact
-          json_keys json_object json_set json_set_exact json_valid len
-          like ln log lower ltrim match max md5 min mvappend mvcount
-          mvdedup mvfilter mvfind mvindex mvjoin mvmap mvrange mvsort
-          mvzip mv_to_json_array now null nullif pi pow printf random
-          relative_time replace round rtrim searchmatch sha1 sha256
-          sha512 sigfig sin sinh split sqrt strftime strptime substr sum
-          tan tanh time toarray tobool todouble toint tomv tonumber
-          toobject tostring trim true typeof upper urldecode validate
+          json_keys json_object json_set json_set_exact json_valid
+          len like ln log lower ltrim match max md5 memk min mktime mstime
+          mv_to_json_array mvappend mvcount mvdedup mvfilter mvfind mvindex
+          mvjoin mvmap mvrange mvreverse mvsort mvzip
+          now null nullif pi pow printf random relative_time replace rmcomma
+          rmunit round rtrim searchmatch sha1 sha256 sha512 sigfig sin sinh
+          split sqrt spath strftime strptime substr sum tan tanh time toarray
+          tobool todouble toint tomv tonumber toobject tostring trim true typeof
+          upper urldecode validate
         )
       end
 
-      # Statistical and charting functions (from Splunk stats functions reference)
+      # Statistical and charting functions
+      # Sources:
+      #   Aggregate:   https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/statistical-and-charting-functions/aggregate-functions
+      #   Event order: https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/statistical-and-charting-functions/event-order-functions
+      #   Multivalue:  https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/statistical-and-charting-functions/multivalue-stats-and-chart-functions
+      #   Time:        https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/statistical-and-charting-functions/time-functions
       def self.stats_functions
         @stats_functions ||= Set.new %w(
-          avg count distinct_count dc estdc estdc_error exactperc max
-          mean median min mode perc percentile range stdev stdevp sum
-          sumsq upperperc var varp first last list values earliest
-          earliest_time latest latest_time per_day per_hour per_minute
-          per_second rate rate_avg rate_sum sparkline
+          avg average c count dc distinct_count earliest earliest_time
+          estdc estdc_error first last latest latest_time list max mean
+          median min mode per_day per_hour per_minute per_second range rate
+          rate_avg rate_sum sparkline stdev stdevp sum sumsq upperperc
+          values var varp
         )
       end
 
       # Operator keywords
+      # Source: https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/introduction/understanding-spl-syntax
+      #         https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/search-commands/replace
+      #         https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/search-commands/lookup
       def self.operator_words
         @operator_words ||= Set.new %w(
-          AND OR NOT XOR IN LIKE BY AS OVER OUTPUT OUTPUTNEW WHERE
+          AND AS BY IN LIKE NOT OR OUTPUT OUTPUTNEW OVER WHERE WITH XOR
         )
       end
 
-      # Constants
+      # Boolean and null constants
       def self.constants
         @constants ||= Set.new %w(
-          true false TRUE FALSE null NULL
+          false FALSE null NULL true TRUE
         )
       end
 
-      # Built-in / internal fields
+      # Built-in and internal fields
+      # Sources: Splunk internal metadata fields;
+      #          https://help.splunk.com/en/splunk-enterprise/search/spl-search-reference/10.2/search-commands/addinfo
       def self.builtin_fields
         @builtin_fields ||= Set.new %w(
-          _time _raw _indextime _cd _serial _bkt _si _sourcetype
-          _subsecond _kv host source sourcetype index splunk_server
-          linecount punct timeendpos timestartpos eventtype tag
-          date_hour date_mday date_minute date_month date_second
-          date_wday date_year date_zone
+          _bkt _cd _indextime _kv _raw _serial _si _sourcetype _subsecond _time
+          date_hour date_mday date_minute date_month date_second date_wday
+          date_year date_zone eventtype host index info_max_time info_min_time
+          info_search_time info_sid linecount punct source sourcetype
+          splunk_server tag timeendpos timestartpos
         )
       end
 
@@ -107,65 +137,72 @@ module Rouge
         # Block comments (triple backtick)
         rule %r/```/, Comment::Multiline, :block_comment
 
-        # Single-line comments (starting with ` followed by content)
-        # SPL doesn't have single-line comments in the traditional sense
-
         # Double-quoted strings
         rule %r/"/, Str::Double, :double_string
 
-        # Single-quoted strings (field names)
+        # Single-quoted strings (field names / literal values)
         rule %r/'/, Str::Single, :single_string
 
-        # Backtick-quoted macros/saved searches (not triple)
+        # Backtick macro references (not triple backtick)
         rule %r/`(?!``)/, Name::Function, :backtick_string
 
-        # Numeric literals
-        rule %r/-?\d+\.\d+(?:e[+-]?\d+)?/i, Num::Float
-        rule %r/-?\d+(?:e[+-]?\d+)?/i, Num::Integer
+        # Macro argument substitution: $varname$
+        rule %r/\$\w+\$/, Name::Variable
 
-        # Time modifiers (e.g., -24h@h, +7d@d, -30m, now)
-        rule %r/[+-]\d+[smhdwqy](?:@[smhdwqy])?/i, Literal::Date
+        # Numeric literals — floats before integers to avoid partial matches
+        rule %r/\d+\.\d+(?:e[+-]?\d+)?/i, Num::Float
+        rule %r/\d+(?:e[+-]?\d+)?/i,       Num::Integer
 
-        # Subsearch brackets
-        rule %r/[\[\]]/, Punctuation
+        # Brackets and braces
+        rule %r/[\[\]{}]/, Punctuation
 
         # Pipe operator
         rule %r/\|/, Punctuation
 
-        # Comparison and assignment operators
-        rule %r/[<>!=]=?/, Operator
+        # Comparison operators (longest match first)
+        rule %r/[<>!]=|[<>]/, Operator
         rule %r/==/, Operator
 
-        # Arithmetic and string concatenation operators
+        # Arithmetic and string-concatenation operators
         rule %r/[+\-*\/%]/, Operator
         rule %r/\.\./, Operator
-        rule %r/\.(?!\w)/, Operator
+        rule %r/\./, Operator
+
+        # Time snap operator and other special characters
+        rule %r/@/, Operator
+        rule %r/:/, Punctuation
+        rule %r/\$/, Operator
 
         # Other punctuation
         rule %r/[(),;]/, Punctuation
 
-        # Equals sign (assignment / field=value)
+        # Assignment / field=value
         rule %r/=/, Operator
 
         # Wildcard
         rule %r/\*/, Operator
 
+        # Functions with required numeric suffix:
+        #   Percentile:  perc90, p95, exactperc99, upperperc90
+        #   Trendline:   sma5, ema10, wma20 (Source: https://help.splunk.com/.../search-commands/trendline)
+        rule %r/(?:exactperc|upperperc|perc|sma|ema|wma|p)\d+\b/, Name::Builtin
+
         # Words — classify by set membership
         rule %r/\w+/ do |m|
-          word = m[0]
-          word_upper = word.upcase
+          word       = m[0]
           word_lower = word.downcase
-          if self.class.constants.include? word
+          word_upper = word.upcase
+          if self.class.constants.include?(word)
             token Keyword::Constant
-          elsif self.class.operator_words.include? word_upper
+          elsif self.class.operator_words.include?(word_upper)
             token Keyword::Pseudo
-          elsif self.class.commands.include? word_lower
+          elsif self.class.commands.include?(word_lower)
             token Keyword
-          elsif self.class.eval_functions.include? word_lower
+          elsif self.class.eval_functions.include?(word_lower)
             token Name::Builtin
-          elsif self.class.stats_functions.include? word_lower
+          elsif self.class.stats_functions.include?(word_lower)
             token Name::Builtin
-          elsif self.class.builtin_fields.include? word_lower
+          elsif self.class.builtin_fields.include?(word_lower)
             token Name::Variable::Magic
           else
             token Name
@@ -174,25 +211,25 @@ module Rouge
       end
 
       state :block_comment do
-        rule %r/```/, Comment::Multiline, :pop!
-        rule %r/[^`]+/, Comment::Multiline
-        rule %r/`/, Comment::Multiline
+        rule %r/```/,    Comment::Multiline, :pop!
+        rule %r/[^`]+/,  Comment::Multiline
+        rule %r/`/,      Comment::Multiline
       end
 
       state :double_string do
-        rule %r/\\./, Str::Escape
-        rule %r/"/, Str::Double, :pop!
-        rule %r/[^\\"]+/, Str::Double
+        rule %r/\\./,      Str::Escape
+        rule %r/"/,        Str::Double, :pop!
+        rule %r/[^\\"]+/,  Str::Double
       end
 
       state :single_string do
-        rule %r/\\./, Str::Escape
-        rule %r/'/, Str::Single, :pop!
-        rule %r/[^\\']+/, Str::Single
+        rule %r/\\./,      Str::Escape
+        rule %r/'/,        Str::Single, :pop!
+        rule %r/[^\\']+/,  Str::Single
       end
 
       state :backtick_string do
-        rule %r/`/, Name::Function, :pop!
+        rule %r/`/,     Name::Function, :pop!
         rule %r/[^`]+/, Name::Function
       end
     end

data/spec/visual/samples/spl

@@ -96,9 +96,13 @@ index=main earliest=-30m latest=now
 
 | eval rand_num=random()
 
+| eval rounded_up=ceil(3.2)
+| eval reversed=mvreverse(myfield)
+
 index=main sourcetype=access_combined
 | stats count AS total_requests,
     avg(response_time) AS avg_response,
+    average(response_time) AS avg_response2,
     max(response_time) AS max_response,
     min(response_time) AS min_response,
     dc(clientip) AS unique_clients,
@@ -114,7 +118,8 @@ index=main sourcetype=access_combined
     values(method) AS methods,
     earliest(_time) AS first_seen,
     latest(_time) AS last_seen,
-    perc95(response_time) AS p95
+    perc95(response_time) AS p95,
+    sparkline count
   BY host, sourcetype
 
 index=main
@@ -175,7 +180,8 @@ index=main sourcetype=access_combined
 | where status IN (200, 301, 404, 500)
 
 | convert timeformat="%Y-%m-%d" ctime(_time) AS event_date
-| convert rmunit(percent_cpu)
+| convert rmunit(percent_cpu) dur2sec(response_duration) rmcomma(revenue)
+| convert memk(memory_used) mstime(elapsed_time) mktime(event_date_str) AS epoch
 
 status!=200 AND (method="POST" OR method="PUT") NOT url="*/health*"
 
@@ -194,14 +200,19 @@ true false TRUE FALSE null NULL
 
 _time _raw host source sourcetype index
 
+| addinfo
+_time info_min_time info_max_time info_sid info_search_time
+
 | transaction host maxspan=5m maxpause=30s
 | iplocation clientip
 | geostats latfield=lat longfield=lon count BY status
 
 | predict response_time AS predicted_response future_timespan=24
-| trendline sma5(response_time) AS trend
+| trendline sma5(response_time) AS sma_trend ema10(response_time) AS ema_trend wma20(response_time) AS wma_trend
 | anomalydetection field=response_time
 
+| noop
+
 | makeresults count=10
 | eval random_val=random() % 100
 | outputlookup my_lookup.csv
@@ -219,7 +230,7 @@ index=main
 | makemv delim="," values
 | mvexpand values
 
-| foreach * [eval <<FIELD>>=if(isnull(<<FIELD>>), 0, <<FIELD>>)]
+| foreach bytes_* [eval total=total+bytes_out]
 | eval result=printf("%s has %d errors (%.2f%%)", host, count, pct)
 | strcat source ":" sourcetype full_source
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge-lexer-spl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sean Whalen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-03-06 00:00:00.000000000 Z
+date: 2026-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rouge
@@ -31,6 +31,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
 - lib/rouge/lexer/spl.rb
 - lib/rouge/lexers/spl.rb
 - spec/demos/spl
@@ -41,6 +42,8 @@ licenses:
 metadata:
   source_code_uri: https://github.com/seanthegeek/rouge-lexer-spl
   bug_tracker_uri: https://github.com/seanthegeek/rouge-lexer-spl/issues
+  changelog_uri: https://github.com/seanthegeek/rouge-lexer-spl/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/seanthegeek/rouge-lexer-spl/blob/main/README.md
 post_install_message:
 rdoc_options: []
 require_paths: