rouge 0.2.0 → 0.2.1

data/lib/rouge/lexers/python.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class Python < RegexLexer
+      desc "The Python programming language (python.org)"
       tag 'python'
       aliases 'py'
       filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'

data/lib/rouge/lexers/ruby.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class Ruby < RegexLexer
+      desc "The Ruby programming language (ruby-lang.org)"
       tag 'ruby'
       aliases 'rb'
       filenames '*.rb', '*.ruby', '*.rbw', '*.rake', '*.gemspec',
@@ -142,7 +143,7 @@ module Rouge
           (module)
           (\s+)
           ([a-zA-Z_][a-zA-Z0-9_]*(::[a-zA-Z_][a-zA-Z0-9_]*)*)
-        ) do
+        )x do
           group 'Keyword'
           group 'Text'
           group 'Name.Namespace'

data/lib/rouge/lexers/scheme.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Scheme < RegexLexer
+      desc "The Scheme variant of Lisp"
+
       tag 'scheme'
       filenames '*.scm', '*.ss', '*.rkt'
       mimetypes 'text/x-scheme', 'application/x-scheme'

data/lib/rouge/lexers/shell.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Shell < RegexLexer
+      desc "Various shell languages, including sh and bash"
+
       tag 'shell'
       aliases 'bash', 'zsh', 'ksh', 'sh'
       filenames '*.sh', '*.bash', '*.zsh', '*.ksh'

data/lib/rouge/lexers/sql.rb

@@ -0,0 +1,137 @@
+module Rouge
+  module Lexers
+    class SQL < RegexLexer
+      desc "Structured Query Language, for relational databases"
+      tag 'sql'
+      filenames '*.sql'
+      mimetypes 'text/x-sql'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS
+          ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE
+          ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION
+          AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH
+          BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED
+          CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS
+          CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME
+          CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT
+          CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE
+          COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA
+          COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE
+          COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER
+          CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS
+          CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA
+          CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY
+          CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE
+          CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME
+          CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA
+          DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION
+          DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED
+          DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE
+          DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS
+          DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP
+          DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE
+          ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT
+          ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS
+          EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE
+          FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION
+          G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP
+          GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE
+          ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING
+          INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY
+          INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD
+          INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY
+          KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL
+          LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL
+          LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH
+          MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT
+          METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH
+          MORE MOVE MUMPS NAMES NATIONAL NATURAL NCHAR NCLOB NEW NEXT
+          NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL
+          NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS
+          OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER
+          ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING
+          OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME
+          PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG
+          PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL
+          PENDANT PLACING PLI POSITION POSTFIX PRECISION PREFIX PREORDER
+          PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE
+          PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING
+          REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART
+          RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH
+          RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
+          ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS
+          ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL
+          SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE
+          SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW
+          SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE
+          SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE
+          SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS
+          STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN
+          SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER
+          TABLE TABLE_NAME  TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN
+          TIMESTAMP TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING
+          TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK
+          TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION
+          TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM
+          TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION
+          UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER
+          USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME
+          USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES
+          VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE
+          WITH WITHOUT WORK WRITE YEAR ZONE
+        )
+      end
+
+      state :root do
+        rule /\s+/m, 'Text'
+        rule /--.*?\n/, 'Comment.Single'
+        rule %r(/\*), 'Comment.Multiline', :multiline_comments
+        rule /\d+/, 'Literal.Number.Integer'
+        rule /'/, 'Literal.String.Single', :single_string
+        rule /"/, 'Name.Variable', :double_string
+        rule /`/, 'Name.Variable', :backtick
+
+        rule /\w[\w\d]*/ do |m|
+          if self.class.keywords.include? m[0].upcase
+            token 'Keyword'
+          else
+            token 'Name'
+          end
+        end
+
+        rule %r([+*/<>=~!@#%^&|?^-]), 'Operator'
+        rule /[;:()\[\],.]/, 'Punctuation'
+      end
+
+      state :multiline_comments do
+        rule %r(/[*]), 'Comment.Multiline', :multiline_comments
+        rule %r([*]/), 'Comment.Multiline', :pop!
+        rule %r([^/*]+), 'Comment.Multiline'
+        rule %r([/*]), 'Comment.Multiline'
+      end
+
+      state :backtick do
+        rule /\\./, 'Literal.String.Escape'
+        rule /``/, 'Literal.String.Escape'
+        rule /`/, 'Name.Variable', :pop!
+        rule /[^\\`]+/, 'Name.Variable'
+      end
+
+      state :single_string do
+        rule /\\./, 'Literal.String.Escape'
+        rule /''/, 'Literal.String.Escape'
+        rule /'/, 'Literal.String.Single', :pop!
+        rule /[^\\']+/, 'Literal.String.Single'
+      end
+
+      state :double_string do
+        rule /\\./, 'Literal.String.Escape'
+        rule /""/, 'Literal.String.Escape'
+        rule /"/, 'Name.Variable', :pop!
+        rule /[^\\"]+/, 'Name.Variable'
+      end
+    end
+  end
+end

data/lib/rouge/lexers/tcl.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class TCL < RegexLexer
+      desc "The Tool Command Language (tcl.tk)"
       tag 'tcl'
       filenames '*.tcl'
       mimetypes 'text/x-tcl', 'text/x-script.tcl', 'application/x-tcl'

data/lib/rouge/lexers/tex.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class TeX < RegexLexer
+      desc "The TeX typesetting system"
       tag 'tex'
       aliases 'TeX', 'LaTeX', 'latex'
 

data/lib/rouge/lexers/text.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Text < Lexer
+      desc "A boring lexer that doesn't highlight anything"
+
       tag 'text'
       filenames '*.txt'
       mimetypes 'text/plain'

data/lib/rouge/lexers/viml.rb

@@ -0,0 +1,98 @@
+module Rouge
+  module Lexers
+    class VimL < RegexLexer
+      desc "VimL, the scripting language for the Vim editor (vim.org)"
+      tag 'viml'
+      aliases 'vim', 'vimscript', 'ex'
+      filenames '*.vim', '*.vba', '.vimrc', '.exrc', '.gvimrc',
+                '_vimrc', '_exrc', '_gvimrc' # _ names for windows
+
+      mimetypes 'text/x-vim'
+
+      def self.keywords
+        load Pathname.new(__FILE__).dirname.join('viml/keywords.rb')
+        self.keywords
+      end
+
+      state :root do
+        rule /^(\s*)(".*?)$/ do
+          group 'Text'; group 'Comment'
+        end
+
+        rule /^\s*\\/, 'Literal.String.Escape'
+
+        rule /[ \t]+/, 'Text'
+
+        # TODO: regexes can have other delimiters
+        rule %r(/(\\\\|\\/|[^\n/])*/), 'Literal.String.Regex'
+        rule %r("(\\\\|\\"|[^\n"])*"), 'Literal.String.Double'
+        rule %r('(\\\\|\\'|[^\n'])*'), 'Literal.String.Single'
+
+        # if it's not a string, it's a comment.
+        rule /(?<=\s)"[^-:.%#=*].*?$/, 'Comment'
+
+        rule /-?\d+/, 'Literal.Number'
+        rule /#[0-9a-f]{6}/i, 'Literal.Number.Hex'
+        rule /^:/, 'Punctuation'
+        rule /[():<>+=!\[\]{}\|,~.-]/, 'Punctuation'
+        rule /\b(let|if|else|endif|elseif|fun|function|endfunction)\b/,
+          'Keyword'
+
+        rule /\b(NONE|bold|italic|underline|dark|light)\b/, 'Name.Builtin'
+
+        rule /[absg]:\w+\b/, 'Name.Variable'
+        rule /\b\w+\b/, 'Postprocess.Name'
+
+        # no errors in VimL!
+        rule /./m, 'Text'
+      end
+
+      postprocess 'Postprocess.Name' do |tok, name|
+        keywords = self.class.keywords
+
+        if mapping_contains?(keywords[:command], name)
+          token 'Keyword', name
+        elsif mapping_contains?(keywords[:option], name)
+          token 'Name.Builtin', name
+        elsif mapping_contains?(keywords[:auto], name)
+          token 'Name.Builtin', name
+        else
+          token 'Text', name
+        end
+      end
+
+      def mapping_contains?(mapping, word)
+        shortest, longest = find_likely_mapping(mapping, word)
+
+        word.start_with?(shortest) and longest.start_with?(word)
+      end
+
+      # binary search through the mappings to find the one that's likely
+      # to actually work.
+      def find_likely_mapping(mapping, word)
+        min = 0
+        max = mapping.size
+
+        until max == min
+          mid = (max + min) / 2
+
+          cmp, _ = mapping[mid]
+
+          case word <=> cmp
+          when 1
+            # too low
+            min = mid + 1
+          when -1
+            # too high
+            max = mid
+          when 0
+            # just right, abort!
+            return mapping[mid]
+          end
+        end
+
+        mapping[max - 1]
+      end
+    end
+  end
+end

data/lib/rouge/lexers/viml/keywords.rb

@@ -0,0 +1,11 @@
+# DO NOT EDIT: automatically generated by `rake vimkeywords`.
+# see tasks/vim.rake for more info.
+module Rouge
+  module Lexers
+    class VimL
+      def self.keywords
+        @keywords ||= {:command=>[[":p", ":p"], ["Allargs", "Allargs"], ["DiffOrig", "DiffOrig"], ["Error", "Error"], ["Man", "Man"], ["MyCommand", "MyCommand"], ["Mycmd", "Mycmd"], ["N", "N"], ["N", "Next"], ["P", "P"], ["P", "Print"], ["Ren", "Ren"], ["Rena", "Rena"], ["Renu", "Renu"], ["TOhtml", "TOhtml"], ["X", "X"], ["XMLent", "XMLent"], ["XMLns", "XMLns"], ["a", "a"], ["ab", "ab"], ["abc", "abclear"], ["abo", "aboveleft"], ["al", "all"], ["ar", "ar"], ["ar", "args"], ["arga", "argadd"], ["argd", "argdelete"], ["argdo", "argdo"], ["arge", "argedit"], ["argg", "argglobal"], ["argl", "arglocal"], ["argu", "argument"], ["as", "ascii"], ["au", "au"], ["b", "buffer"], ["bN", "bNext"], ["ba", "ball"], ["bad", "badd"], ["bar", "bar"], ["bd", "bdelete"], ["bel", "belowright"], ["bf", "bfirst"], ["bl", "blast"], ["bm", "bmodified"], ["bn", "bnext"], ["bo", "botright"], ["bp", "bprevious"], ["br", "br"], ["br", "brewind"], ["brea", "break"], ["breaka", "breakadd"], ["breakd", "breakdel"], ["breakl", "breaklist"], ["bro", "browse"], ["browseset", "browseset"], ["bu", "bu"], ["buf", "buf"], ["bufdo", "bufdo"], ["buffers", "buffers"], ["bun", "bunload"], ["bw", "bwipeout"], ["c", "c"], ["c", "change"], ["cN", "cN"], ["cN", "cNext"], ["cNf", "cNf"], ["cNf", "cNfile"], ["cabc", "cabclear"], ["cad", "cad"], ["cad", "caddexpr"], ["caddb", "caddbuffer"], ["caddf", "caddfile"], ["cal", "call"], ["cat", "catch"], ["cb", "cbuffer"], ["cc", "cc"], ["ccl", "cclose"], ["cd", "cd"], ["ce", "center"], ["cex", "cexpr"], ["cf", "cfile"], ["cfir", "cfirst"], ["cg", "cgetfile"], ["cgetb", "cgetbuffer"], ["cgete", "cgetexpr"], ["changes", "changes"], ["chd", "chdir"], ["che", "checkpath"], ["checkt", "checktime"], ["cl", "cl"], ["cl", "clist"], ["cla", "clast"], ["clo", "close"], ["cmapc", "cmapclear"], ["cmdname", "cmdname"], ["cn", "cn"], ["cn", "cnext"], ["cnew", "cnewer"], ["cnf", "cnf"], ["cnf", "cnfile"], ["co", "copy"], ["col", "colder"], ["colo", "colorscheme"], ["com", "com"], ["comc", "comclear"], ["comment", "comment"], ["comp", "compiler"], ["con", "con"], ["con", "continue"], ["conf", "confirm"], ["cope", "copen"], ["count", "count"], ["cp", "cprevious"], ["cpf", "cpfile"], ["cq", "cquit"], ["cr", "crewind"], ["cs", "cs"], ["cscope", "cscope"], ["cstag", "cstag"], ["cuna", "cunabbrev"], ["cw", "cwindow"], ["d", "d"], ["d", "delete"], ["de", "de"], ["debug", "debug"], ["debugg", "debuggreedy"], ["del", "del"], ["delc", "delcommand"], ["delf", "delf"], ["delf", "delfunction"], ["delm", "delmarks"], ["di", "di"], ["di", "display"], ["diffg", "diffget"], ["diffo", "diffo"], ["diffoff", "diffoff"], ["diffp", "diffp"], ["diffpatch", "diffpatch"], ["diffpu", "diffput"], ["diffsplit", "diffsplit"], ["difft", "difft"], ["diffthis", "diffthis"], ["diffu", "diffupdate"], ["dig", "dig"], ["dig", "digraphs"], ["dj", "djump"], ["dl", "dlist"], ["do", "do"], ["doau", "doau"], ["dr", "drop"], ["ds", "dsearch"], ["dsp", "dsplit"], ["dwim", "dwim"], ["e", "e"], ["e", "edit"], ["e:e", "e:e"], ["e:e:e", "e:e:e"], ["e:e:r", "e:e:r"], ["ea", "ea"], ["earlier", "earlier"], ["ec", "ec"], ["echoe", "echoerr"], ["echom", "echomsg"], ["echon", "echon"], ["el", "else"], ["elsei", "elseif"], ["em", "emenu"], ["emenu", "emenu"], ["en", "en"], ["en", "endif"], ["endf", "endf"], ["endf", "endfunction"], ["endfo", "endfor"], ["endfun", "endfun"], ["endt", "endtry"], ["endw", "endwhile"], ["ene", "enew"], ["ex", "ex"], ["exi", "exit"], ["exu", "exusage"], ["f", "f"], ["f", "file"], ["filename", "filename"], ["files", "files"], ["filet", "filet"], ["filetype", "filetype"], ["fin", "fin"], ["fin", "find"], ["fina", "finally"], ["fini", "finish"], ["fir", "first"], ["fix", "fixdel"], ["fo", "fold"], ["foldc", "foldclose"], ["foldd", "folddoopen"], ["folddoc", "folddoclosed"], ["foldo", "foldopen"], ["for", "for"], ["fu", "fu"], ["fu", "function"], ["fun", "fun"], ["g", "g"], ["get", "get"], ["go", "goto"], ["gr", "grep"], ["grepa", "grepadd"], ["gs", "gs"], ["gui", "gui"], ["gvim", "gvim"], ["h", "h"], ["h", "help"], ["ha", "hardcopy"], ["helpf", "helpfind"], ["helpg", "helpgrep"], ["helpt", "helptags"], ["hi", "hi"], ["hid", "hide"], ["his", "history"], ["i", "i"], ["ia", "ia"], ["iabc", "iabclear"], ["if", "if"], ["ij", "ijump"], ["il", "ilist"], ["imapc", "imapclear"], ["in", "in"], ["index", "index"], ["intro", "intro"], ["is", "isearch"], ["isp", "isplit"], ["iuna", "iunabbrev"], ["j", "join"], ["ju", "jumps"], ["k", "k"], ["kee", "keepmarks"], ["keepa", "keepa"], ["keepalt", "keepalt"], ["keepj", "keepjumps"], ["l", "l"], ["l", "list"], ["lN", "lN"], ["lN", "lNext"], ["lNf", "lNf"], ["lNf", "lNfile"], ["la", "la"], ["la", "last"], ["lad", "lad"], ["lad", "laddexpr"], ["laddb", "laddbuffer"], ["laddf", "laddfile"], ["lan", "lan"], ["lan", "language"], ["lat", "lat"], ["later", "later"], ["lb", "lbuffer"], ["lc", "lcd"], ["lch", "lchdir"], ["lcl", "lclose"], ["lcs", "lcs"], ["lcscope", "lcscope"], ["le", "left"], ["lefta", "leftabove"], ["let", "let"], ["lex", "lexpr"], ["lf", "lfile"], ["lfir", "lfirst"], ["lg", "lgetfile"], ["lgetb", "lgetbuffer"], ["lgete", "lgetexpr"], ["lgr", "lgrep"], ["lgrepa", "lgrepadd"], ["lh", "lhelpgrep"], ["ll", "ll"], ["lla", "llast"], ["lli", "llist"], ["lmak", "lmake"], ["lmapc", "lmapclear"], ["lne", "lne"], ["lne", "lnext"], ["lnew", "lnewer"], ["lnf", "lnf"], ["lnf", "lnfile"], ["lo", "lo"], ["lo", "loadview"], ["loadk", "loadk"], ["loadkeymap", "loadkeymap"], ["loc", "lockmarks"], ["locale", "locale"], ["lockv", "lockvar"], ["lol", "lolder"], ["lop", "lopen"], ["lp", "lprevious"], ["lpf", "lpfile"], ["lr", "lrewind"], ["ls", "ls"], ["lt", "ltag"], ["lua", "lua"], ["luado", "luado"], ["luafile", "luafile"], ["lv", "lvimgrep"], ["lvimgrepa", "lvimgrepadd"], ["lw", "lwindow"], ["m", "move"], ["ma", "ma"], ["ma", "mark"], ["main", "main"], ["main", "main"], ["mak", "make"], ["marks", "marks"], ["mat", "match"], ["menut", "menut"], ["menut", "menutranslate"], ["mes", "mes"], ["messages", "messages"], ["mk", "mk"], ["mk", "mkexrc"], ["mkdir", "mkdir"], ["mks", "mksession"], ["mksp", "mkspell"], ["mkv", "mkv"], ["mkv", "mkvimrc"], ["mkvie", "mkview"], ["mo", "mo"], ["mod", "mode"], ["mv", "mv"], ["mz", "mz"], ["mz", "mzscheme"], ["mzf", "mzfile"], ["n", "n"], ["n", "n"], ["n", "next"], ["nb", "nbkey"], ["nbc", "nbclose"], ["nbs", "nbstart"], ["ne", "ne"], ["new", "new"], ["nkf", "nkf"], ["nmapc", "nmapclear"], ["noa", "noa"], ["noautocmd", "noautocmd"], ["noh", "nohlsearch"], ["nu", "number"], ["o", "o"], ["o", "open"], ["ol", "oldfiles"], ["omapc", "omapclear"], ["on", "only"], ["opt", "options"], ["ownsyntax", "ownsyntax"], ["p", "p"], ["p", "print"], ["p:", "p:"], ["p:", "p:"], ["p:gs", "p:gs"], ["p:h", "p:h"], ["p:h:h", "p:h:h"], ["p:r", "p:r"], ["p:t", "p:t"], ["pat", "pat"], ["pat", "pat"], ["pc", "pclose"], ["pe", "pe"], ["pe", "perl"], ["ped", "pedit"], ["perld", "perldo"], ["po", "pop"], ["popu", "popu"], ["popu", "popup"], ["pp", "ppop"], ["pr", "pr"], ["pre", "preserve"], ["prev", "previous"], ["pro", "pro"], ["prof", "profile"], ["profd", "profdel"], ["promptf", "promptfind"], ["promptr", "promptrepl"], ["ps", "psearch"], ["ptN", "ptN"], ["ptN", "ptNext"], ["pta", "ptag"], ["ptf", "ptfirst"], ["ptj", "ptjump"], ["ptl", "ptlast"], ["ptn", "ptn"], ["ptn", "ptnext"], ["ptp", "ptprevious"], ["ptr", "ptrewind"], ["pts", "ptselect"], ["pu", "put"], ["pw", "pwd"], ["py", "py"], ["py", "python"], ["py3", "py3"], ["py3", "py3"], ["py3file", "py3file"], ["pyf", "pyfile"], ["python3", "python3"], ["q", "q"], ["q", "quit"], ["qa", "qall"], ["quita", "quitall"], ["quote", "quote"], ["r", "r"], ["r", "r"], ["r", "read"], ["r:e", "r:e"], ["r:r", "r:r"], ["r:r:r", "r:r:r"], ["re", "re"], ["rec", "recover"], ["red", "red"], ["red", "redo"], ["redi", "redir"], ["redr", "redraw"], ["redraws", "redrawstatus"], ["reg", "registers"], ["res", "resize"], ["ret", "retab"], ["retu", "return"], ["rew", "rewind"], ["ri", "right"], ["rightb", "rightbelow"], ["ru", "ru"], ["ru", "runtime"], ["rub", "ruby"], ["rubyd", "rubydo"], ["rubyf", "rubyfile"], ["rundo", "rundo"], ["rv", "rviminfo"], ["s", "s"], ["s", "s"], ["s", "s"], ["s", "s"], ["sN", "sNext"], ["sa", "sargument"], ["sal", "sall"], ["san", "sandbox"], ["sav", "saveas"], ["sb", "sbuffer"], ["sbN", "sbNext"], ["sba", "sball"], ["sbf", "sbfirst"], ["sbl", "sblast"], ["sbm", "sbmodified"], ["sbn", "sbnext"], ["sbp", "sbprevious"], ["sbr", "sbrewind"], ["scrip", "scrip"], ["scrip", "scriptnames"], ["scripte", "scriptencoding"], ["scs", "scs"], ["scscope", "scscope"], ["se", "set"], ["setf", "setfiletype"], ["setg", "setglobal"], ["setl", "setlocal"], ["sf", "sfind"], ["sfir", "sfirst"], ["sh", "shell"], ["si", "si"], ["sig", "sig"], ["sign", "sign"], ["sil", "silent"], ["sim", "simalt"], ["sl", "sl"], ["sl", "sleep"], ["sla", "slast"], ["sm", "smagic"], ["sm", "smap"], ["sme", "sme"], ["smenu", "smenu"], ["sn", "snext"], ["sni", "sniff"], ["sno", "snomagic"], ["snoreme", "snoreme"], ["snoremenu", "snoremenu"], ["so", "so"], ["so", "source"], ["sor", "sort"], ["sp", "split"], ["spe", "spe"], ["spe", "spellgood"], ["spelld", "spelldump"], ["spelli", "spellinfo"], ["spellr", "spellrepall"], ["spellu", "spellundo"], ["spellw", "spellwrong"], ["spr", "sprevious"], ["sre", "srewind"], ["st", "st"], ["st", "stop"], ["sta", "stag"], ["star", "star"], ["star", "startinsert"], ["start", "start"], ["startg", "startgreplace"], ["startr", "startreplace"], ["stj", "stjump"], ["stopi", "stopinsert"], ["sts", "stselect"], ["sub", "sub"], ["sub", "sub"], ["sun", "sunhide"], ["sunme", "sunme"], ["sunmenu", "sunmenu"], ["sus", "suspend"], ["sv", "sview"], ["sw", "swapname"], ["sy", "sy"], ["syn", "syn"], ["sync", "sync"], ["syncbind", "syncbind"], ["synlist", "synlist"], ["t", "t"], ["t:r", "t:r"], ["tN", "tN"], ["tN", "tNext"], ["ta", "ta"], ["ta", "tag"], ["tab", "tab"], ["tabN", "tabN"], ["tabN", "tabNext"], ["tabc", "tabclose"], ["tabd", "tabdo"], ["tabe", "tabedit"], ["tabf", "tabfind"], ["tabfir", "tabfirst"], ["tabl", "tablast"], ["tabm", "tabmove"], ["tabn", "tabnext"], ["tabnew", "tabnew"], ["tabo", "tabonly"], ["tabp", "tabprevious"], ["tabr", "tabrewind"], ["tabs", "tabs"], ["tags", "tags"], ["tc", "tcl"], ["tcld", "tcldo"], ["tclf", "tclfile"], ["te", "tearoff"], ["tf", "tfirst"], ["th", "throw"], ["tj", "tjump"], ["tl", "tlast"], ["tm", "tm"], ["tm", "tmenu"], ["tn", "tn"], ["tn", "tnext"], ["to", "topleft"], ["tp", "tprevious"], ["tr", "tr"], ["tr", "trewind"], ["try", "try"], ["ts", "tselect"], ["tu", "tu"], ["tu", "tunmenu"], ["u", "u"], ["u", "undo"], ["un", "un"], ["una", "unabbreviate"], ["undoj", "undojoin"], ["undol", "undolist"], ["unh", "unhide"], ["unl", "unl"], ["unlo", "unlockvar"], ["uns", "unsilent"], ["up", "update"], ["v", "v"], ["ve", "ve"], ["ve", "version"], ["verb", "verbose"], ["version", "version"], ["version", "version"], ["vert", "vertical"], ["vi", "vi"], ["vi", "visual"], ["vie", "view"], ["vim", "vimgrep"], ["vimgrepa", "vimgrepadd"], ["viu", "viusage"], ["vmapc", "vmapclear"], ["vne", "vnew"], ["vs", "vsplit"], ["w", "w"], ["w", "write"], ["wN", "wNext"], ["wa", "wall"], ["wh", "while"], ["win", "win"], ["win", "winsize"], ["winc", "wincmd"], ["windo", "windo"], ["winp", "winpos"], ["wn", "wnext"], ["wp", "wprevious"], ["wq", "wq"], ["wqa", "wqall"], ["ws", "wsverb"], ["wundo", "wundo"], ["wv", "wviminfo"], ["x", "x"], ["x", "xit"], ["xa", "xall"], ["xmapc", "xmapclear"], ["xme", "xme"], ["xmenu", "xmenu"], ["xnoreme", "xnoreme"], ["xnoremenu", "xnoremenu"], ["xterm", "xterm"], ["xunme", "xunme"], ["xunmenu", "xunmenu"], ["xwininfo", "xwininfo"], ["y", "yank"]], :option=>[], :auto=>[["BufAdd", "BufAdd"], ["BufCreate", "BufCreate"], ["BufDelete", "BufDelete"], ["BufEnter", "BufEnter"], ["BufFilePost", "BufFilePost"], ["BufFilePre", "BufFilePre"], ["BufHidden", "BufHidden"], ["BufLeave", "BufLeave"], ["BufNew", "BufNew"], ["BufNewFile", "BufNewFile"], ["BufRead", "BufRead"], ["BufReadCmd", "BufReadCmd"], ["BufReadPost", "BufReadPost"], ["BufReadPre", "BufReadPre"], ["BufUnload", "BufUnload"], ["BufWinEnter", "BufWinEnter"], ["BufWinLeave", "BufWinLeave"], ["BufWipeout", "BufWipeout"], ["BufWrite", "BufWrite"], ["BufWriteCmd", "BufWriteCmd"], ["BufWritePost", "BufWritePost"], ["BufWritePre", "BufWritePre"], ["Cmd", "Cmd"], ["CmdwinEnter", "CmdwinEnter"], ["CmdwinLeave", "CmdwinLeave"], ["ColorScheme", "ColorScheme"], ["CursorHold", "CursorHold"], ["CursorHoldI", "CursorHoldI"], ["CursorMoved", "CursorMoved"], ["CursorMovedI", "CursorMovedI"], ["EncodingChanged", "EncodingChanged"], ["FileAppendCmd", "FileAppendCmd"], ["FileAppendPost", "FileAppendPost"], ["FileAppendPre", "FileAppendPre"], ["FileChangedRO", "FileChangedRO"], ["FileChangedShell", "FileChangedShell"], ["FileChangedShellPost", "FileChangedShellPost"], ["FileEncoding", "FileEncoding"], ["FileReadCmd", "FileReadCmd"], ["FileReadPost", "FileReadPost"], ["FileReadPre", "FileReadPre"], ["FileType", "FileType"], ["FileWriteCmd", "FileWriteCmd"], ["FileWritePost", "FileWritePost"], ["FileWritePre", "FileWritePre"], ["FilterReadPost", "FilterReadPost"], ["FilterReadPre", "FilterReadPre"], ["FilterWritePost", "FilterWritePost"], ["FilterWritePre", "FilterWritePre"], ["FocusGained", "FocusGained"], ["FocusLost", "FocusLost"], ["FuncUndefined", "FuncUndefined"], ["GUIEnter", "GUIEnter"], ["GUIFailed", "GUIFailed"], ["InsertChange", "InsertChange"], ["InsertCharPre", "InsertCharPre"], ["InsertEnter", "InsertEnter"], ["InsertLeave", "InsertLeave"], ["MenuPopup", "MenuPopup"], ["QuickFixCmdPost", "QuickFixCmdPost"], ["QuickFixCmdPre", "QuickFixCmdPre"], ["RemoteReply", "RemoteReply"], ["SessionLoadPost", "SessionLoadPost"], ["ShellCmdPost", "ShellCmdPost"], ["ShellFilterPost", "ShellFilterPost"], ["SourceCmd", "SourceCmd"], ["SourcePre", "SourcePre"], ["SpellFileMissing", "SpellFileMissing"], ["StdinReadPost", "StdinReadPost"], ["StdinReadPre", "StdinReadPre"], ["SwapExists", "SwapExists"], ["Syntax", "Syntax"], ["TabEnter", "TabEnter"], ["TabLeave", "TabLeave"], ["TermChanged", "TermChanged"], ["TermResponse", "TermResponse"], ["User", "User"], ["UserGettingBored", "UserGettingBored"], ["VimEnter", "VimEnter"], ["VimLeave", "VimLeave"], ["VimLeavePre", "VimLeavePre"], ["VimResized", "VimResized"], ["WinEnter", "WinEnter"], ["WinLeave", "WinLeave"], ["event", "event"]]}
+      end
+    end
+  end
+end

data/lib/rouge/lexers/xml.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class XML < RegexLexer
+      desc %q(<desc for="this-lexer">XML</desc>)
       tag 'xml'
       filenames *%w(*.xml *.xsl *.rss *.xslt *.xsd *.wsdl)
       mimetypes *%w(

data/lib/rouge/lexers/yaml.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class YAML < RegexLexer
+      desc "Yaml Ain't Markup Language (yaml.org)"
       tag 'yaml'
       aliases 'yml'
 
@@ -123,7 +124,7 @@ module Rouge
       # indented line in the block context
       state :block_line do
         # line end
-          rule /[ ]*(?=#|$)/, 'Text', :pop!
+        rule /[ ]*(?=#|$)/, 'Text', :pop!
         rule /[ ]+/, 'Text'
         # tags, anchors, and aliases
         mixin :descriptors
@@ -145,8 +146,8 @@ module Rouge
 
         # a tag in the form '!', '!suffix' or '!handle!suffix'
         rule %r(
-          !(?:[\w-]+)? #handle
-          (?:![\w;/?:@&=+$,.!~*\'()\[\]%-]+)? #suffix
+          (?:![\w-]+)? # handle
+          !(?:[\w;/?:@&=+$,.!~*\'()\[\]%-]*) # suffix
         )x, 'Keyword.Type'
 
         # an anchor

data/lib/rouge/regex_lexer.rb

@@ -1,13 +1,18 @@
 module Rouge
+  # @abstract
+  # A stateful lexer that uses sets of regular expressions to
+  # tokenize a string.  Most lexers are instances of RegexLexer.
   class RegexLexer < Lexer
+    # A rule is a tuple of a regular expression to test, and a callback
+    # to perform if the test succeeds.
+    #
+    # @see StateDSL#rule
     class Rule
       attr_reader :callback
-      attr_reader :next_state
       attr_reader :re
-      def initialize(re, callback, next_state)
+      def initialize(re, callback)
         @re = re
         @callback = callback
-        @next_state = next_state
       end
 
       def inspect
@@ -15,22 +20,17 @@ module Rouge
       end
     end
 
+    # a State is a named set of rules that can be tested for or
+    # mixed in.
+    #
+    # @see RegexLexer.state
     class State
       attr_reader :name
-      def initialize(lexer_class, name, &defn)
-        @lexer_class = lexer_class
+      def initialize(name, &defn)
         @name = name
         @defn = defn
       end
 
-      def relative_state(state_name=nil, &b)
-        if state_name
-          @lexer_class.get_state(state_name)
-        else
-          State.new(@lexer_class, b.inspect, &b).load!
-        end
-      end
-
       def rules
         @rules ||= []
       end
@@ -49,6 +49,25 @@ module Rouge
         @rules = rules
       end
 
+      # Define a new rule for this state.
+      #
+      # @overload rule(re, token, next_state=nil)
+      # @overload rule(re, &callback)
+      #
+      # @param [Regexp] re
+      #   a regular expression for this rule to test.
+      # @param [String] tok
+      #   the token type to yield if `re` matches.
+      # @param [#to_s] next_state
+      #   (optional) a state to push onto the stack if `re` matches.
+      #   If `next_state` is `:pop!`, the state stack will be popped
+      #   instead.
+      # @param [Proc] callback
+      #   a block that will be evaluated in the context of the lexer
+      #   if `re` matches.  This block has access to a number of lexer
+      #   methods, including {RegexLexer#push}, {RegexLexer#pop!},
+      #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
+      #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
         if block_given?
           next_state = tok
@@ -66,22 +85,29 @@ module Rouge
           end
         end
 
-        rules << Rule.new(re, callback, next_state)
+        rules << Rule.new(re, callback)
       end
 
+      # Mix in the rules from another state into this state.  The rules
+      # from the mixed-in state will be tried in order before moving on
+      # to the rest of the rules in this state.
       def mixin(lexer_name)
         rules << lexer_name.to_s
       end
     end
 
+    # The states hash for this lexer.
+    # @see state
     def self.states
       @states ||= {}
     end
 
-    @start_procs = []
+    # The routines to run at the beginning of a fresh lex.
+    # @see start
     def self.start_procs
       @start_procs ||= InheritableList.new(superclass.start_procs)
     end
+    @start_procs = []
 
     # Specify an action to be run every fresh lex.
     #
@@ -91,20 +117,33 @@ module Rouge
       start_procs << b
     end
 
+    # Specify a filter to be applied as the lexer yields tokens.
+    #
+    # @param toktype
+    #   The token type to postprocess
+    # @yield [tok, val]
+    #   The token and the matched value.  The block will be evaluated in
+    #   the context of the lexer, and it must yield an equivalent
+    #   token/value pair, usually by calling #token.
     def self.postprocess(toktype, &b)
       postprocesses << [Token[toktype], b]
     end
 
-    @postprocesses = []
+    # where the postprocess blocks are stored.
+    # @see postprocess
     def self.postprocesses
       @postprocesses ||= InheritableList.new(superclass.postprocesses)
     end
+    @postprocesses = []
 
+    # Define a new state for this lexer with the given name.
+    # The block will be evaluated in the context of a {StateDSL}.
     def self.state(name, &b)
       name = name.to_s
-      states[name] = State.new(self, name, &b)
+      states[name] = State.new(name, &b)
     end
 
+    # @private
     def self.get_state(name)
       return name if name.is_a? State
 
@@ -113,41 +152,55 @@ module Rouge
       state.load!
     end
 
-    def self.[](name)
-      get_state(name)
-    end
-
-    def get_state(name)
-      self.class.get_state(name)
+    # @private
+    def get_state(state_name)
+      self.class.get_state(state_name)
     end
 
+    # The state stack.  This is initially the single state `[:root]`.
+    # It is an error for this stack to be empty.
+    # @see #state
     def stack
       @stack ||= [get_state(:root)]
     end
 
+    # The current state - i.e. one on top of the state stack.
+    #
+    # NB: if the state stack is empty, this will throw an error rather
+    # than returning nil.
     def state
       stack.last or raise 'empty stack!'
     end
 
+    # reset this lexer to its initial state.  This runs all of the
+    # start_procs.
     def reset!
-      @scan_state = nil
+      @stack = nil
 
       self.class.start_procs.each do |pr|
         instance_eval(&pr)
       end
     end
 
+    # This implements the lexer protocol, by yielding [token, value] pairs.
+    #
+    # The process for lexing works as follows, until the stream is empty:
+    #
+    # 1. We look at the state on top of the stack (which by default is
+    #    `[:root]`).
+    # 2. Each rule in that state is tried until one is successful.  If one
+    #    is found, that rule's callback is evaluated - which may yield
+    #    tokens and manipulate the state stack.  Otherwise, one character
+    #    is consumed with an `'Error'` token, and we continue at (1.)
+    #
+    # @see #step #step (where (2.) is implemented)
     def stream_tokens(stream, &b)
       stream_without_postprocessing(stream) do |tok, val|
         _, processor = self.class.postprocesses.find { |t, _| t == tok }
 
         if processor
-          # TODO: DRY this up with run_callback
-          Enumerator.new do |y|
-            @output_stream = y
+          with_output_stream(b) do
             instance_exec(tok, val, &processor)
-          end.each do |newtok, newval|
-            yield Token[newtok], newval
           end
         else
           yield tok, val
@@ -170,6 +223,11 @@ module Rouge
       end
     end
 
+    # Runs one step of the lex.  Rules in the current state are tried
+    # until one matches, at which point its callback is called.
+    #
+    # @return true if a rule was tried successfully
+    # @return false otherwise.
     def step(state, stream, &b)
       state.rules.each do |rule|
         return true if run_rule(rule, stream, &b)
@@ -178,6 +236,7 @@ module Rouge
       false
     end
 
+    # @private
     def run_rule(rule, stream, &b)
       case rule
       when String
@@ -190,26 +249,26 @@ module Rouge
         scan(stream, rule.re) do
           debug { "    got #{stream[0].inspect}" }
 
-          run_callback(stream, &rule.callback).each do |tok, res|
-            debug { "    yielding #{tok.to_s.inspect}, #{res.inspect}" }
-            b.call(Token[tok], res)
-          end
+          run_callback(stream, rule.callback, &b)
         end
       end
     end
 
-    def run_callback(stream, &callback)
-      Enumerator.new do |y|
-        @output_stream = y
+    # @private
+    def run_callback(stream, callback, &output_stream)
+      with_output_stream(output_stream) do
         @group_count = 0
         @last_match = stream
         instance_exec(stream, &callback)
         @last_match = nil
-        @output_stream = nil
       end
     end
 
-    MAX_NULL_STEPS = 5
+    # The number of successive scans permitted without consuming
+    # the input stream.  If this is exceeded, the match fails.
+    MAX_NULL_SCANS = 5
+
+    # @private
     def scan(scanner, re, &b)
       # XXX HACK XXX
       # StringScanner's implementation of ^ is b0rken.
@@ -219,7 +278,7 @@ module Rouge
 
       @null_steps ||= 0
 
-      if @null_steps >= MAX_NULL_STEPS
+      if @null_steps >= MAX_NULL_SCANS
         debug { "    too many scans without consuming the string!" }
         return false
       end
@@ -240,6 +299,13 @@ module Rouge
       return false
     end
 
+    # Yield a token.
+    #
+    # @param tok
+    #   the token type
+    # @param val
+    #   (optional) the string value to yield.  If absent, this defaults
+    #   to the entire last match.
     def token(tok, val=:__absent__)
       val = @last_match[0] if val == :__absent__
       val ||= ''
@@ -249,10 +315,21 @@ module Rouge
       @output_stream << [Token[tok], val] unless val.empty?
     end
 
+    # Yield a token with the next matched group.  Subsequent calls
+    # to this method will yield subsequent groups.
     def group(tok)
       token(tok, @last_match[@group_count += 1])
     end
 
+    # Delegate the lex to another lexer.  The #lex method will be called
+    # with `:continue` set to true, so that #reset! will not be called.
+    # In this way, a single lexer can be repeatedly delegated to while
+    # maintaining its own internal state stack.
+    #
+    # @param [#lex] lexer
+    #   The lexer or lexer class to delegate to
+    # @param [String] text
+    #   The text to delegate.  This defaults to the last matched string.
     def delegate(lexer, text=nil)
       debug { "    delegating to #{lexer.inspect}" }
       text ||= @last_match[0]
@@ -263,18 +340,25 @@ module Rouge
       end
     end
 
+    # Push a state onto the stack.  If no state name is given and you've
+    # passed a block, a state will be dynamically created using the
+    # {StateDSL}.
     def push(state_name=nil, &b)
-      # use the top of the stack by default
-      if state_name || b
-        push_state = state.relative_state(state_name, &b)
+      push_state = if state_name
+        get_state(state_name)
+      elsif block_given?
+        State.new(b.inspect, &b).load!
       else
-        push_state = self.state
+        # use the top of the stack by default
+        self.state
       end
 
       debug { "    pushing #{push_state.name}" }
       stack.push(push_state)
     end
 
+    # Pop the state stack.  If a number is passed in, it will be popped
+    # that number of times.
     def pop!(times=1)
       raise 'empty stack!' if stack.empty?
 
@@ -282,19 +366,34 @@ module Rouge
       times.times { stack.pop }
     end
 
+    # reset the stack back to `[:root]`.
     def reset_stack
       debug { '    resetting stack' }
       stack.clear
       stack.push get_state(:root)
     end
 
+    # Check if `state_name` is in the state stack.
     def in_state?(state_name)
       stack.map(&:name).include? state_name.to_s
     end
 
+    # Check if `state_name` is the state on top of the state stack.
     def state?(state_name)
       state_name.to_s == state.name
     end
 
+  private
+    def with_output_stream(output_stream, &b)
+      @output_stream = Yielder.new do |tok, val|
+        debug { "    yielding #{tok.to_s.inspect}, #{val.inspect}" }
+        output_stream.call(Token[tok], val)
+      end
+
+      yield
+
+    ensure
+      @output_stream = nil
+    end
   end
 end