rouge 3.17.0

data/lib/rouge/lexers/escape.rb

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Escape < Lexer
+      tag 'escape'
+      aliases 'esc'
+
+      desc 'A generic lexer for including escaped content - see Formatter.enable_escape!'
+
+      option :start, 'the beginning of the escaped section, default "<!"'
+      option :end, 'the end of the escaped section, e.g. "!>"'
+      option :lang, 'the language to lex in unescaped sections'
+
+      attr_reader :start
+      attr_reader :end
+      attr_reader :lang
+
+      def initialize(*)
+        super
+        @start = string_option(:start) { '<!' }
+        @end = string_option(:end) { '!>' }
+        @lang = lexer_option(:lang) { PlainText.new }
+      end
+
+      def to_start_regex
+        @to_start_regex ||= /(.*?)(#{Regexp.escape(@start)})/m
+      end
+
+      def to_end_regex
+        @to_end_regex ||= /(.*?)(#{Regexp.escape(@end)})/m
+      end
+
+      def stream_tokens(str, &b)
+        stream = StringScanner.new(str)
+
+        loop do
+          if stream.scan(to_start_regex)
+            puts "pre-escape: #{stream[1].inspect}" if @debug
+            @lang.continue_lex(stream[1], &b)
+          else
+            # no more start delimiters, scan til the end
+            @lang.continue_lex(stream.rest, &b)
+            return
+          end
+
+          if stream.scan(to_end_regex)
+            yield Token::Tokens::Escape, stream[1]
+          else
+            yield Token::Tokens::Escape, stream.rest
+            return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/factor.rb

@@ -0,0 +1,303 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Factor < RegexLexer
+      title "Factor"
+      desc "Factor, the practical stack language (factorcode.org)"
+      tag 'factor'
+      filenames '*.factor'
+      mimetypes 'text/x-factor'
+
+      def self.detect?(text)
+        return true if text.shebang? 'factor'
+      end
+
+      def self.builtins
+        @builtins ||= {}.tap do |builtins|
+          builtins[:kernel] = Set.new %w(
+            or 2bi 2tri while wrapper nip 4dip wrapper? bi*
+            callstack>array both? hashcode die dupd callstack
+            callstack? 3dup tri@ pick curry build ?execute 3bi prepose
+            >boolean if clone eq? tri* ? = swapd 2over 2keep 3keep clear
+            2dup when not tuple? dup 2bi* 2tri* call tri-curry object bi@
+            do unless* if* loop bi-curry* drop when* assert= retainstack
+            assert? -rot execute 2bi@ 2tri@ boa with either? 3drop bi
+            curry?  datastack until 3dip over 3curry tri-curry* tri-curry@
+            swap and 2nip throw bi-curry (clone) hashcode* compose 2dip if
+            3tri unless compose? tuple keep 2curry equal? assert tri 2drop
+            most <wrapper> boolean? identity-hashcode identity-tuple?
+            null new dip bi-curry@ rot xor identity-tuple boolean
+          )
+
+          builtins[:assocs] = Set.new %w(
+            ?at assoc? assoc-clone-like assoc= delete-at* assoc-partition
+            extract-keys new-assoc value? assoc-size map>assoc push-at
+            assoc-like key? assoc-intersect assoc-refine update
+            assoc-union assoc-combine at* assoc-empty? at+ set-at
+            assoc-all? assoc-subset?  assoc-hashcode change-at assoc-each
+            assoc-diff zip values value-at rename-at inc-at enum? at cache
+            assoc>map <enum> assoc assoc-map enum value-at* assoc-map-as
+            >alist assoc-filter-as clear-assoc assoc-stack maybe-set-at
+            substitute assoc-filter 2cache delete-at assoc-find keys
+            assoc-any? unzip
+          )
+
+          builtins[:combinators] = Set.new %w(
+            case execute-effect no-cond no-case? 3cleave>quot 2cleave
+            cond>quot wrong-values? no-cond? cleave>quot no-case case>quot
+            3cleave wrong-values to-fixed-point alist>quot case-find
+            cond cleave call-effect 2cleave>quot recursive-hashcode
+            linear-case-quot spread spread>quot
+          )
+
+          builtins[:math] = Set.new %w(
+            number= if-zero next-power-of-2 each-integer ?1+
+            fp-special? imaginary-part unless-zero float>bits number?
+            fp-infinity? bignum? fp-snan? denominator fp-bitwise= *
+            + power-of-2? - u>= / >= bitand log2-expects-positive <
+            log2 > integer? number bits>double 2/ zero? (find-integer)
+            bits>float float? shift ratio? even? ratio fp-sign bitnot
+            >fixnum complex? /i /f byte-array>bignum when-zero sgn >bignum
+            next-float u< u> mod recip rational find-last-integer >float
+            (all-integers?) 2^ times integer fixnum? neg fixnum sq bignum
+            (each-integer) bit? fp-qnan? find-integer complex <fp-nan>
+            real double>bits bitor rem fp-nan-payload all-integers?
+            real-part log2-expects-positive? prev-float align unordered?
+            float fp-nan? abs bitxor u<= odd? <= /mod rational? >integer
+            real? numerator
+          )
+
+          builtins[:sequences] = Set.new %w(
+            member-eq? append assert-sequence= find-last-from
+            trim-head-slice clone-like 3sequence assert-sequence? map-as
+            last-index-from reversed index-from cut* pad-tail
+            remove-eq! concat-as but-last snip trim-tail nths
+            nth 2selector sequence slice?  <slice> partition
+            remove-nth tail-slice empty? tail* if-empty
+            find-from virtual-sequence? member? set-length
+            drop-prefix unclip unclip-last-slice iota map-sum
+            bounds-error? sequence-hashcode-step selector-for
+            accumulate-as map start midpoint@ (accumulate) rest-slice
+            prepend fourth sift accumulate! new-sequence follow map! like
+            first4 1sequence reverse slice unless-empty padding virtual@
+            repetition? set-last index 4sequence max-length set-second
+            immutable-sequence first2 first3 replicate-as reduce-index
+            unclip-slice supremum suffix! insert-nth trim-tail-slice
+            tail 3append short count suffix concat flip filter sum
+            immutable? reverse! 2sequence map-integers delete-all start*
+            indices snip-slice check-slice sequence?  head map-find
+            filter! append-as reduce sequence= halves collapse-slice
+            interleave 2map filter-as binary-reduce slice-error? product
+            bounds-check? bounds-check harvest immutable virtual-exemplar
+            find produce remove pad-head last replicate set-fourth
+            remove-eq shorten reversed?  map-find-last 3map-as
+            2unclip-slice shorter? 3map find-last head-slice pop* 2map-as
+            tail-slice* but-last-slice 2map-reduce iota? collector-for
+            accumulate each selector append! new-resizable cut-slice
+            each-index head-slice* 2reverse-each sequence-hashcode
+            pop set-nth ?nth <flat-slice> second join when-empty
+            collector immutable-sequence? <reversed> all? 3append-as
+            virtual-sequence subseq? remove-nth! push-either new-like
+            length last-index push-if 2all? lengthen assert-sequence
+            copy map-reduce move third first 3each tail? set-first prefix
+            bounds-error any? <repetition> trim-slice exchange surround
+            2reduce cut change-nth min-length set-third produce-as
+            push-all head? delete-slice rest sum-lengths 2each head*
+            infimum remove! glue slice-error subseq trim replace-slice
+            push repetition map-index trim-head unclip-last mismatch
+          )
+
+          builtins[:namespaces] = Set.new %w(
+            global +@ change set-namestack change-global init-namespaces
+            on off set-global namespace set with-scope bind with-variable
+            inc dec counter initialize namestack get get-global make-assoc
+          )
+
+          builtins[:arrays] = Set.new %w(
+            <array> 2array 3array pair >array 1array 4array pair?
+            array resize-array array?
+          )
+
+          builtins[:io] = Set.new %w(
+            +character+ bad-seek-type? readln each-morsel
+            stream-seek read print with-output-stream contents
+            write1 stream-write1 stream-copy stream-element-type
+            with-input-stream stream-print stream-read stream-contents
+            stream-tell tell-output bl seek-output bad-seek-type nl
+            stream-nl write flush stream-lines +byte+ stream-flush
+            read1 seek-absolute? stream-read1 lines stream-readln
+            stream-read-until each-line seek-end with-output-stream*
+            seek-absolute with-streams seek-input seek-relative?
+            input-stream stream-write read-partial seek-end?
+            seek-relative error-stream read-until with-input-stream*
+            with-streams* tell-input each-block output-stream
+            stream-read-partial each-stream-block each-stream-line
+          )
+
+          builtins[:strings] = Set.new %w(
+            resize-string >string <string> 1string string string?
+          )
+
+          builtins[:vectors] = Set.new %w(
+            with-return restarts return-continuation with-datastack
+            recover rethrow-restarts <restart> ifcc set-catchstack
+            >continuation< cleanup ignore-errors restart?
+            compute-restarts attempt-all-error error-thread
+            continue <continuation> attempt-all-error? condition?
+            <condition> throw-restarts error catchstack continue-with
+            thread-error-hook continuation rethrow callcc1
+            error-continuation callcc0 attempt-all condition
+            continuation? restart return
+          )
+
+          builtins[:continuations] = Set.new %w(
+            with-return restarts return-continuation with-datastack
+            recover rethrow-restarts <restart> ifcc set-catchstack
+            >continuation< cleanup ignore-errors restart?
+            compute-restarts attempt-all-error error-thread
+            continue <continuation> attempt-all-error? condition?
+            <condition> throw-restarts error catchstack continue-with
+            thread-error-hook continuation rethrow callcc1
+            error-continuation callcc0 attempt-all condition
+            continuation? restart return
+          )
+        end
+      end
+
+      state :root do
+        rule %r/\s+/m, Text
+
+        rule %r/(:|::|MACRO:|MEMO:|GENERIC:|HELP:)(\s+)(\S+)/m do
+          groups Keyword, Text, Name::Function
+        end
+
+        rule %r/(M:|HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)/m do
+          groups Keyword, Text, Name::Class, Text, Name::Function
+        end
+
+        rule %r/\((?=\s)/, Name::Function, :stack_effect
+        rule %r/;(?=\s)/, Keyword
+
+        rule %r/(USING:)((?:\s|\\\s)+)/m do
+          groups Keyword::Namespace, Text
+          push :import
+        end
+
+        rule %r/(IN:|USE:|UNUSE:|QUALIFIED:|QUALIFIED-WITH:)(\s+)(\S+)/m do
+          groups Keyword::Namespace, Text, Name::Namespace
+        end
+
+        rule %r/(FROM:|EXCLUDE:)(\s+)(\S+)(\s+)(=>)/m do
+          groups Keyword::Namespace, Text, Name::Namespace, Text, Punctuation
+        end
+
+        rule %r/(?:ALIAS|DEFER|FORGET|POSTPONE):/, Keyword::Namespace
+
+        rule %r/(TUPLE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+          groups(
+            Keyword, Text,
+            Name::Class, Text,
+            Punctuation, Text,
+            Name::Class
+          )
+          push :slots
+        end
+
+        rule %r/(TUPLE:)(\s+)(\S+)/m do
+          groups Keyword, Text, Name::Class
+          push :slots
+        end
+
+        rule %r/(UNION:|INTERSECTION:)(\s+)(\S+)/m do
+          groups Keyword, Text, Name::Class
+        end
+
+        rule %r/(PREDICATE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
+          groups(
+            Keyword, Text,
+            Name::Class, Text,
+            Punctuation, Text,
+            Name::Class
+          )
+        end
+
+        rule %r/(C:)(\s+)(\S+)(\s+)(\S+)/m do
+          groups(
+            Keyword, Text,
+            Name::Function, Text,
+            Name::Class
+          )
+        end
+
+        rule %r(
+          (INSTANCE|SLOT|MIXIN|SINGLETONS?|CONSTANT|SYMBOLS?|ERROR|SYNTAX
+           |ALIEN|TYPEDEF|FUNCTION|STRUCT):
+        )x, Keyword
+
+        rule %r/(?:<PRIVATE|PRIVATE>)/, Keyword::Namespace
+
+        rule %r/(MAIN:)(\s+)(\S+)/ do
+          groups Keyword::Namespace, Text, Name::Function
+        end
+
+        # strings
+        rule %r/"""\s+.*?\s+"""/, Str
+        rule %r/"(\\.|[^\\])*?"/, Str
+        rule %r/(CHAR:)(\s+)(\\[\\abfnrstv]*|\S)(?=\s)/, Str::Char
+
+        # comments
+        rule %r/!\s+.*$/, Comment
+        rule %r/#!\s+.*$/, Comment
+
+        # booleans
+        rule %r/[tf](?=\s)/, Name::Constant
+
+        # numbers
+        rule %r/-?\d+\.\d+(?=\s)/, Num::Float
+        rule %r/-?\d+(?=\s)/, Num::Integer
+
+        rule %r/HEX:\s+[a-fA-F\d]+(?=\s)/m, Num::Hex
+        rule %r/BIN:\s+[01]+(?=\s)/, Num::Bin
+        rule %r/OCT:\s+[0-7]+(?=\s)/, Num::Oct
+
+        rule %r([-+/*=<>^](?=\s)), Operator
+
+        rule %r/(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/,
+          Keyword
+
+        rule %r/\S+/ do |m|
+          name = m[0]
+
+          if self.class.builtins.values.any? { |b| b.include? name }
+            token Name::Builtin
+          else
+            token Name
+          end
+        end
+      end
+
+      state :stack_effect do
+        rule %r/\s+/, Text
+        rule %r/\(/, Name::Function, :stack_effect
+        rule %r/\)/, Name::Function, :pop!
+
+        rule %r/--/, Name::Function
+        rule %r/\S+/, Name::Variable
+      end
+
+      state :slots do
+        rule %r/\s+/, Text
+        rule %r/;(?=\s)/, Keyword, :pop!
+        rule %r/\S+/, Name::Variable
+      end
+
+      state :import do
+        rule %r/;(?=\s)/, Keyword, :pop!
+        rule %r/\s+/, Text
+        rule %r/\S+/, Name::Namespace
+      end
+    end
+  end
+end

data/lib/rouge/lexers/fortran.rb

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+# vim: set ts=2 sw=2 et:
+
+# TODO: Implement format list support.
+
+module Rouge
+  module Lexers
+    class Fortran < RegexLexer
+      title "Fortran"
+      desc "Fortran 2008 (free-form)"
+
+      tag 'fortran'
+      filenames '*.f', '*.f90', '*.f95', '*.f03', '*.f08',
+                '*.F', '*.F90', '*.F95', '*.F03', '*.F08'
+      mimetypes 'text/x-fortran'
+
+      name = /[A-Z][_A-Z0-9]*/i
+      kind_param = /(\d+|#{name})/
+      exponent = /[ED][+-]?\d+/i
+
+      def self.keywords
+        # Special rules for two-word keywords are defined further down.
+        # Note: Fortran allows to omit whitespace between certain keywords.
+        @keywords ||= Set.new %w(
+          abstract allocatable allocate assign assignment associate asynchronous
+          backspace bind block blockdata call case class close codimension
+          common concurrent contains contiguous continue critical cycle data
+          deallocate deferred dimension do elemental else elseif elsewhere end
+          endassociate endblock endblockdata enddo endenum endfile endforall
+          endfunction endif endinterface endmodule endprogram endselect
+          endsubmodule endsubroutine endtype endwhere endwhile entry enum
+          enumerator equivalence exit extends external final flush forall format
+          function generic goto if implicit import in include inout inquire
+          intent interface intrinsic is lock module namelist non_overridable
+          none nopass nullify only open operator optional out parameter pass
+          pause pointer print private procedure program protected public pure
+          read recursive result return rewind save select selectcase sequence
+          stop submodule subroutine target then type unlock use value volatile
+          wait where while write
+        )
+      end
+
+      def self.types
+        # A special rule for the two-word version "double precision" is
+        # defined further down.
+        @types ||= Set.new %w(
+          character complex doubleprecision integer logical real
+        )
+      end
+
+      def self.intrinsics
+        @intrinsics ||= Set.new %w(
+          abs achar acos acosh adjustl adjustr aimag aint all allocated anint
+          any asin asinh associated atan atan2 atanh atomic_define atomic_ref
+          bessel_j0 bessel_j1 bessel_jn bessel_y0 bessel_y1 bessel_yn bge bgt
+          bit_size ble blt btest c_associated c_f_pointer c_f_procpointer
+          c_funloc c_loc c_sizeof ceiling char cmplx command_argument_count
+          compiler_options compiler_version conjg cos cosh count cpu_time cshift
+          date_and_time dble digits dim dot_product dprod dshiftl dshiftr
+          eoshift epsilon erf erfc_scaled erfc execute_command_line exp exponent
+          extends_type_of findloc floor fraction gamma get_command_argument
+          get_command get_environment_variable huge hypot iachar iall iand iany
+          ibclr ibits ibset ichar ieee_class ieee_copy_sign ieee_get_flag
+          ieee_get_halting_mode ieee_get_rounding_mode ieee_get_status
+          ieee_get_underflow_mode ieee_is_finite ieee_is_nan ieee_is_normal
+          ieee_logb ieee_next_after ieee_rem ieee_rint ieee_scalb
+          ieee_selected_real_kind ieee_set_flag ieee_set_halting_mode
+          ieee_set_rounding_mode ieee_set_status ieee_set_underflow_mode
+          ieee_support_datatype ieee_support_denormal ieee_support_divide
+          ieee_support_flag ieee_support_halting ieee_support_inf
+          ieee_support_io ieee_support_nan ieee_support_rounding
+          ieee_support_sqrt ieee_support_standard ieee_support_underflow_control
+          ieee_unordered ieee_value ieor image_index index int ior iparity
+          is_contiguous is_iostat_end is_iostat_eor ishft ishftc kind lbound
+          lcobound leadz len_trim len lge lgt lle llt log_gamma log log10
+          logical maskl maskr matmul max maxexponent maxloc maxval merge_bits
+          merge min minexponent minloc minval mod modulo move_alloc mvbits
+          nearest new_line nint norm2 not null num_images pack parity popcnt
+          poppar present product radix random_number random_seed range real
+          repeat reshape rrspacing same_type_as scale scan selected_char_kind
+          selected_int_kind selected_real_kind set_exponent shape shifta shiftl
+          shiftr sign sin sinh size spacing spread sqrt storage_size sum
+          system_clock tan tanh this_image tiny trailz transfer transpose trim
+          ubound ucobound unpack verify
+        )
+      end
+
+      state :root do
+        rule %r/[\s]+/, Text::Whitespace
+        rule %r/!.*$/, Comment::Single
+        rule %r/^#.*$/, Comment::Preproc
+
+        rule %r/::|[()\/;,:&\[\]]/, Punctuation
+
+        # TODO: This does not take into account line continuation.
+        rule %r/^(\s*)([0-9]+)\b/m do |m|
+          token Text::Whitespace, m[1]
+          token Name::Label, m[2]
+        end
+
+        # Format statements are quite a strange beast.
+        # Better process them in their own state.
+        rule %r/\b(FORMAT)(\s*)(\()/mi do |m|
+          token Keyword, m[1]
+          token Text::Whitespace, m[2]
+          token Punctuation, m[3]
+          push :format_spec
+        end
+
+        rule %r(
+          [+-]? # sign
+          (
+            (\d+[.]\d*|[.]\d+)(#{exponent})?
+            | \d+#{exponent} # exponent is mandatory
+          )
+          (_#{kind_param})? # kind parameter
+        )xi, Num::Float
+
+        rule %r/[+-]?\d+(_#{kind_param})?/i, Num::Integer
+        rule %r/B'[01]+'|B"[01]+"/i, Num::Bin
+        rule %r/O'[0-7]+'|O"[0-7]+"/i, Num::Oct
+        rule %r/Z'[0-9A-F]+'|Z"[0-9A-F]+"/i, Num::Hex
+        rule %r/(#{kind_param}_)?'/, Str::Single, :string_single
+        rule %r/(#{kind_param}_)?"/, Str::Double, :string_double
+        rule %r/[.](TRUE|FALSE)[.](_#{kind_param})?/i, Keyword::Constant
+
+        rule %r{\*\*|//|==|/=|<=|>=|=>|[-+*/<>=%]}, Operator
+        rule %r/\.(?:EQ|NE|LT|LE|GT|GE|NOT|AND|OR|EQV|NEQV|[A-Z]+)\./i, Operator::Word
+
+        # Special rules for two-word keywords and types.
+        # Note: "doubleprecision" is covered by the normal keyword rule.
+        rule %r/double\s+precision\b/i, Keyword::Type
+        rule %r/go\s+to\b/i, Keyword
+        rule %r/sync\s+(all|images|memory)\b/i, Keyword
+        rule %r/error\s+stop\b/i, Keyword
+
+        rule %r/#{name}/m do |m|
+          match = m[0].downcase
+          if self.class.keywords.include? match
+            token Keyword
+          elsif self.class.types.include? match
+            token Keyword::Type
+          elsif self.class.intrinsics.include? match
+            token Name::Builtin
+          else
+            token Name
+          end
+        end
+
+      end
+
+      state :string_single do
+        rule %r/[^']+/, Str::Single
+        rule %r/''/, Str::Escape
+        rule %r/'/, Str::Single, :pop!
+      end
+
+      state :string_double do
+        rule %r/[^"]+/, Str::Double
+        rule %r/""/, Str::Escape
+        rule %r/"/, Str::Double, :pop!
+      end
+
+      state :format_spec do
+        rule %r/'/, Str::Single, :string_single
+        rule %r/"/, Str::Double, :string_double
+        rule %r/\(/, Punctuation, :format_spec
+        rule %r/\)/, Punctuation, :pop!
+        rule %r/,/, Punctuation
+        rule %r/[\s]+/, Text::Whitespace
+        # Edit descriptors could be seen as a kind of "format literal".
+        rule %r/[^\s'"(),]+/, Literal
+      end
+    end
+  end
+end