djot 0.0.5 → 0.0.7

data/lib/lua/djot/inline.lua

@@ -1,641 +0,0 @@
--- this allows the code to work with both lua and luajit:
-local unpack = unpack or table.unpack
-local match = require("djot.match")
-local attributes = require("djot.attributes")
-local make_match, unpack_match, matches_pattern =
-  match.make_match, match.unpack_match, match.matches_pattern
-local find, byte = string.find, string.byte
-
--- allow up to 3 captures...
-local function bounded_find(subj, patt, startpos, endpos)
-  local sp,ep,c1,c2,c3 = find(subj, patt, startpos)
-  if ep and ep <= endpos then
-    return sp,ep,c1,c2,c3
-  end
-end
-
-local Parser = {}
-
-function Parser:new(subject, opts)
-  local state =
-    { opts = opts or {}, -- options
-      subject = subject,
-      matches = {}, -- table pos : (endpos, annotation)
-      warnings = {}, -- array of {pos, string} arrays
-      openers = {}, -- map from closer_type to array of (pos, data) in reverse order
-      verbatim = 0, -- parsing verbatim span to be ended by n backticks
-      verbatim_type = nil, -- whether verbatim is math or regular
-      destination = false, -- parsing link destination in ()
-      firstpos = 0, -- position of first slice
-      lastpos = 0,  -- position of last slice
-      allow_attributes = true, -- allow parsing of attributes
-      attribute_parser = nil,  -- attribute parser
-      attribute_start = nil,  -- start of potential attribute
-      attribute_slices = nil, -- slices we've tried to parse as attributes
-    }
-  setmetatable(state, self)
-  self.__index = self
-  return state
-end
-
-function Parser:add_match(startpos, endpos, annotation)
-  self.matches[startpos] = make_match(startpos, endpos, annotation)
-end
-
-function Parser:add_opener(name, ...)
-  -- 1 = startpos, 2 = endpos, 3 = annotation, 4 = substartpos, 5 = endpos
-  if not self.openers[name] then
-    self.openers[name] = {}
-  end
-  table.insert(self.openers[name], {...})
-end
-
-function Parser:clear_openers(startpos, endpos)
-  -- remove other openers in between the matches
-  for _,v in pairs(self.openers) do
-    local i = #v
-    while v[i] do
-      local sp,ep,_,sp2,ep2 = unpack(v[i])
-      if sp >= startpos and ep <= endpos then
-        v[i] = nil
-      elseif (sp2 and sp2 >= startpos) and (ep2 and ep2 <= endpos) then
-        v[i][3] = nil
-        v[i][4] = nil
-        v[i][5] = nil
-      else
-        break
-      end
-      i = i - 1
-    end
-  end
-end
-
-function Parser:str_matches(startpos, endpos)
-  for i = startpos, endpos do
-    local m = self.matches[i]
-    if m then
-      local sp, ep, annot = unpack_match(m)
-      if annot ~= "str" and annot ~= "escape" then
-        self.matches[i] = make_match(sp, ep, "str")
-      end
-    end
-  end
-end
-
-function Parser.between_matched(c, annotation, defaultmatch, opentest)
-  return function(self, pos)
-    local defaultmatch = defaultmatch or "str"
-    local subject = self.subject
-    local can_open = find(subject, "^%S", pos + 1)
-    local can_close = find(subject, "^%S", pos - 1)
-    local has_open_marker = matches_pattern(self.matches[pos - 1], "^open%_marker")
-    local has_close_marker = byte(subject, pos + 1) == 125 -- }
-    local endcloser = pos
-    local startopener = pos
-
-    if type(opentest) == "function" then
-      can_open = can_open and opentest(self, pos)
-    end
-
-    -- allow explicit open/close markers to override:
-    if has_open_marker then
-      can_open = true
-      can_close = false
-      startopener = pos - 1
-    end
-    if not has_open_marker and has_close_marker then
-      can_close = true
-      can_open = false
-      endcloser = pos + 1
-    end
-
-    if has_open_marker and defaultmatch:match("^right") then
-      defaultmatch = defaultmatch:gsub("^right", "left")
-    elseif has_close_marker and defaultmatch:match("^left") then
-      defaultmatch = defaultmatch:gsub("^left", "right")
-    end
-
-    local openers = self.openers[c]
-    local matched = false
-    if can_close and openers and #openers > 0 then
-       -- check openers for a match
-      local openpos, openposend = unpack(openers[#openers])
-      if openposend ~= pos - 1 then -- exclude empty emph
-        self:clear_openers(openpos, pos)
-        self:add_match(openpos, openposend, "+" .. annotation)
-        self:add_match(pos, endcloser, "-" .. annotation)
-        return endcloser + 1
-      end
-    end
-    -- if we get here, we didn't match an opener
-    if can_open then
-      self:add_opener(c, startopener, pos)
-      self:add_match(startopener, pos, defaultmatch)
-      return pos + 1
-    else
-      self:add_match(pos, endcloser, defaultmatch)
-      return endcloser + 1
-    end
-  end
-end
-
-Parser.matchers = {
-    -- 96 = `
-    [96] = function(self, pos, endpos)
-      local subject = self.subject
-      local _, endchar = bounded_find(subject, "^`*", pos, endpos)
-      if not endchar then
-        return nil
-      end
-      if find(subject, "^%$%$", pos - 2) then
-        self.matches[pos - 2] = nil
-        self.matches[pos - 1] = nil
-        self:add_match(pos - 2, endchar, "+display_math")
-        self.verbatim_type = "display_math"
-      elseif find(subject, "^%$", pos - 1) then
-        self.matches[pos - 1] = nil
-        self:add_match(pos - 1, endchar, "+inline_math")
-        self.verbatim_type = "inline_math"
-      else
-        self:add_match(pos, endchar, "+verbatim")
-        self.verbatim_type = "verbatim"
-      end
-      self.verbatim = endchar - pos + 1
-      return endchar + 1
-    end,
-
-    -- 92 = \
-    [92] = function(self, pos, endpos)
-      local subject = self.subject
-      local _, endchar = bounded_find(subject, "^[ \t]*\r?\n",  pos + 1, endpos)
-      self:add_match(pos, pos, "escape")
-      if endchar then
-        -- see if there were preceding spaces
-        if #self.matches > 0 then
-          local sp, ep, annot = unpack_match(self.matches[#self.matches])
-          if annot == "str" then
-            while subject:byte(ep) == 32 or subject:byte(ep) == 9 do
-              ep = ep -1
-            end
-            if sp == ep then
-              self.matches[#self.matches] = nil
-            else
-              self:add_match(sp, ep, "str")
-            end
-          end
-        end
-        self:add_match(pos + 1, endchar, "hardbreak")
-        return endchar + 1
-      else
-        local _, ec = bounded_find(subject, "^[%p ]", pos + 1, endpos)
-        if not ec then
-          self:add_match(pos, pos, "str")
-          return pos + 1
-        else
-          self:add_match(pos, pos, "escape")
-          if find(subject, "^ ", pos + 1) then
-            self:add_match(pos + 1, ec, "nbsp")
-          else
-            self:add_match(pos + 1, ec, "str")
-          end
-          return ec + 1
-        end
-      end
-    end,
-
-    -- 60 = <
-    [60] = function(self, pos, endpos)
-      local subject = self.subject
-      local starturl, endurl =
-              bounded_find(subject, "^%<[^<>%s]+%>", pos, endpos)
-      if starturl then
-        local is_url = bounded_find(subject, "^%a+:", pos + 1, endurl)
-        local is_email = bounded_find(subject, "^[^:]+%@", pos + 1, endurl)
-        if is_email then
-          self:add_match(starturl, starturl, "+email")
-          self:add_match(starturl + 1, endurl - 1, "str")
-          self:add_match(endurl, endurl, "-email")
-          return endurl + 1
-        elseif is_url then
-          self:add_match(starturl, starturl, "+url")
-          self:add_match(starturl + 1, endurl - 1, "str")
-          self:add_match(endurl, endurl, "-url")
-          return endurl + 1
-        end
-      end
-    end,
-
-    -- 126 = ~
-    [126] = Parser.between_matched('~', 'subscript'),
-
-    -- 94 = ^
-    [94] = Parser.between_matched('^', 'superscript'),
-
-    -- 91 = [
-    [91] = function(self, pos, endpos)
-      local sp, ep = bounded_find(self.subject, "^%^([^]]+)%]", pos + 1, endpos)
-      if sp then -- footnote ref
-        self:add_match(pos, ep, "footnote_reference")
-        return ep + 1
-      else
-        self:add_opener("[", pos, pos)
-        self:add_match(pos, pos, "str")
-        return pos + 1
-      end
-    end,
-
-    -- 93 = ]
-    [93] = function(self, pos, endpos)
-      local openers = self.openers["["]
-      local subject = self.subject
-      if openers and #openers > 0 then
-        local opener = openers[#openers]
-        if opener[3] == "reference_link" then
-          -- found a reference link
-          -- add the matches
-          local subject = self.subject
-          local is_image = bounded_find(subject, "^!", opener[1] - 1, endpos)
-                  and not bounded_find(subject, "^[\\]", opener[1] - 2, endpos)
-          if is_image then
-            self:add_match(opener[1] - 1, opener[1] - 1, "image_marker")
-            self:add_match(opener[1], opener[2], "+imagetext")
-            self:add_match(opener[4], opener[5], "-imagetext")
-          else
-            self:add_match(opener[1], opener[2], "+linktext")
-            self:add_match(opener[4], opener[5], "-linktext")
-          end
-          self:add_match(opener[5], opener[5], "+reference")
-          self:add_match(pos, pos, "-reference")
-          -- convert all matches to str
-          self:str_matches(opener[5] + 1, pos - 1)
-          -- remove from openers
-          self:clear_openers(opener[1], pos)
-          return pos + 1
-        elseif bounded_find(subject, "^%[", pos + 1, endpos) then
-          opener[3] = "reference_link"
-          opener[4] = pos  -- intermediate ]
-          opener[5] = pos + 1  -- intermediate [
-          self:add_match(pos, pos + 1, "str")
-          return pos + 2
-        elseif bounded_find(subject, "^%(", pos + 1, endpos) then
-          self.openers["("] = {} -- clear ( openers
-          opener[3] = "explicit_link"
-          opener[4] = pos  -- intermediate ]
-          opener[5] = pos + 1  -- intermediate (
-          self.destination = true
-          self:add_match(pos, pos + 1, "str")
-          return pos + 2
-        elseif bounded_find(subject, "^%{", pos + 1, endpos) then
-          -- assume this is attributes, bracketed span
-          self:add_match(opener[1], opener[2], "+span")
-          self:add_match(pos, pos, "-span")
-          self:clear_openers(opener[1], pos)
-          return pos + 1
-        end
-      end
-    end,
-
-
-    -- 40 = (
-    [40] = function(self, pos)
-      if not self.destination then return nil end
-      self:add_opener("(", pos, pos)
-      self:add_match(pos, pos, "str")
-      return pos + 1
-    end,
-
-    -- 41 = )
-    [41] = function(self, pos, endpos)
-      if not self.destination then return nil end
-      local parens = self.openers["("]
-      if parens and #parens > 0 and parens[#parens][1] then
-        parens[#parens] = nil -- clear opener
-        self:add_match(pos, pos, "str")
-        return pos + 1
-      else
-        local subject = self.subject
-        local openers = self.openers["["]
-        if openers and #openers > 0
-            and openers[#openers][3] == "explicit_link" then
-          local opener = openers[#openers]
-          local startdest, enddest = opener[5], pos
-          -- we have inline link
-          local is_image = bounded_find(subject, "^!", opener[1] - 1, endpos)
-                 and not bounded_find(subject, "^[\\]", opener[1] - 2, endpos)
-          if is_image then
-            self:add_match(opener[1] - 1, opener[1] - 1, "image_marker")
-            self:add_match(opener[1], opener[2], "+imagetext")
-            self:add_match(opener[4], opener[4], "-imagetext")
-          else
-            self:add_match(opener[1], opener[2], "+linktext")
-            self:add_match(opener[4], opener[4], "-linktext")
-          end
-          self:add_match(startdest, startdest, "+destination")
-          self:add_match(enddest, enddest, "-destination")
-          self.destination = false
-          -- convert all matches to str
-          self:str_matches(opener[5] + 1, pos - 1)
-          -- remove from openers
-          self:clear_openers(opener[2], pos)
-          return enddest + 1
-        end
-      end
-    end,
-
-    -- 95 = _
-    [95] = Parser.between_matched('_', 'emph'),
-
-    -- 42 = *
-    [42] = Parser.between_matched('*', 'strong'),
-
-    -- 123 = {
-    [123] = function(self, pos, endpos)
-      if bounded_find(self.subject, "^[_*~^+='\"-]", pos + 1, endpos) then
-        self:add_match(pos, pos, "open_marker")
-        return pos + 1
-      elseif self.allow_attributes then
-        self.attribute_parser = attributes.AttributeParser:new(self.subject)
-        self.attribute_start = pos
-        self.attribute_slices = {}
-        return pos
-      else
-        self:add_match(pos, pos, "str")
-        return pos + 1
-      end
-    end,
-
-    -- 58 = :
-    [58] = function(self, pos, endpos)
-      local sp, ep = bounded_find(self.subject, "^%:[%w_+-]+%:", pos, endpos)
-      if sp then
-        self:add_match(sp, ep, "emoji")
-        return ep + 1
-      else
-        self:add_match(pos, pos, "str")
-        return pos + 1
-      end
-    end,
-
-    -- 43 = +
-    [43] = Parser.between_matched("+", "insert", "str",
-                           function(self, pos)
-                             return find(self.subject, "^%{", pos - 1) or
-                                    find(self.subject, "^%}", pos + 1)
-                           end),
-
-    -- 61 = =
-    [61] = Parser.between_matched("=", "mark", "str",
-                           function(self, pos)
-                             return find(self.subject, "^%{", pos - 1) or
-                                    find(self.subject, "^%}", pos + 1)
-                           end),
-
-    -- 39 = '
-    [39] = Parser.between_matched("'", "single_quoted", "right_single_quote",
-                           function(self, pos) -- test to open
-                             return pos == 1 or
-                               find(self.subject, "^[%s\"'-([]", pos - 1)
-                             end),
-
-    -- 34 = "
-    [34] = Parser.between_matched('"', "double_quoted", "left_double_quote"),
-
-    -- 45 = -
-    [45] = function(self, pos, endpos)
-      local subject = self.subject
-      local _, ep = find(subject, "^%-*", pos)
-      local hyphens
-      if endpos < ep then
-        hyphens = 1 + endpos - pos
-      else
-        hyphens = 1 + ep - pos
-      end
-      if byte(subject, ep + 1) == 125 then -- }
-        hyphens = hyphens - 1 -- last hyphen is close del
-      end
-      if byte(subject, pos - 1) == 123 or byte(subject, pos + 1) == 125 then
-        return Parser.between_matched("-", "delete")(self, pos, endpos)
-      end
-      -- Try to construct a homogeneous sequence of dashes
-      local all_em = hyphens % 3 == 0
-      local all_en = hyphens % 2 == 0
-      while hyphens > 0 do
-        if all_em then
-          self:add_match(pos, pos + 2, "em_dash")
-          pos = pos + 3
-          hyphens = hyphens - 3
-        elseif all_en then
-          self:add_match(pos, pos + 1, "en_dash")
-          pos = pos + 2
-          hyphens = hyphens - 2
-        elseif hyphens >= 3 and (hyphens % 2 ~= 0 or hyphens > 4) then
-          self:add_match(pos, pos + 2, "em_dash")
-          pos = pos + 3
-          hyphens = hyphens - 3
-        elseif hyphens >= 2 then
-          self:add_match(pos, pos + 1, "en_dash")
-          pos = pos + 2
-          hyphens = hyphens - 2
-        else
-          self:add_match(pos, pos, "str")
-          pos = pos + 1
-          hyphens = hyphens - 1
-        end
-      end
-      return pos
-    end,
-
-    -- 46 = .
-    [46] = function(self, pos, endpos)
-      if bounded_find(self.subject, "^%.%.", pos + 1, endpos) then
-        self:add_match(pos, pos +2, "ellipses")
-        return pos + 3
-      end
-    end
-  }
-
-function Parser:single_char(pos)
-  self:add_match(pos, pos, "str")
-  return pos + 1
-end
-
--- Feed a slice to the parser, updating state.
-function Parser:feed(spos, endpos)
-  local special = "[][\\`{}_*()!<>~^:=+$\r\n'\".-]"
-  local subject = self.subject
-  local matchers = self.matchers
-  local pos
-  if self.firstpos == 0 or spos < self.firstpos then
-    self.firstpos = spos
-  end
-  if self.lastpos == 0 or endpos > self.lastpos then
-    self.lastpos = endpos
-  end
-  pos = spos
-  while pos <= endpos do
-    if self.attribute_parser then
-      local sp = pos
-      local ep2 = bounded_find(subject, special, pos, endpos) or endpos
-      local status, ep = self.attribute_parser:feed(sp, ep2)
-      if status == "done" then
-        local attribute_start = self.attribute_start
-        -- add attribute matches
-        self:add_match(attribute_start, attribute_start, "+attributes")
-        self:add_match(ep, ep, "-attributes")
-        local attr_matches = self.attribute_parser:get_matches()
-        -- add attribute matches
-        for i=1,#attr_matches do
-          self:add_match(unpack_match(attr_matches[i]))
-        end
-        -- restore state to prior to adding attribute parser:
-        self.attribute_parser = nil
-        self.attribute_start = nil
-        self.attribute_slices = nil
-        pos = ep + 1
-      elseif status == "fail" then
-        -- backtrack:
-        local slices = self.attribute_slices
-        self.allow_attributes = false
-        self.attribute_parser = nil
-        self.attribute_start = nil
-        for i=1,#slices do
-          self:feed(unpack(slices[i]))
-        end
-        self.allow_attributes = true
-        self.slices = nil
-        pos = sp
-      elseif status == "continue" then
-        self.attribute_slices[#self.attribute_slices + 1] = {sp,ep}
-        pos = ep + 1
-      end
-    else
-      -- find next interesting character:
-      local newpos = bounded_find(subject, special, pos, endpos) or endpos + 1
-      if newpos > pos then
-        self:add_match(pos, newpos - 1, "str")
-        pos = newpos
-        if pos > endpos then
-          break -- otherwise, fall through:
-        end
-      end
-      -- if we get here, then newpos = pos,
-      -- i.e. we have something interesting at pos
-      local c = byte(subject, pos)
-
-      if c == 13 or c == 10 then -- cr or lf
-        if c == 13 and bounded_find(subject, "^[%n]", pos + 1, endpos) then
-          self:add_match(pos, pos + 1, "softbreak")
-          pos = pos + 2
-        else
-          self:add_match(pos, pos, "softbreak")
-          pos = pos + 1
-        end
-      elseif self.verbatim > 0 then
-        if c == 96 then
-          local _, endchar = bounded_find(subject, "^`+", pos, endpos)
-          if endchar and endchar - pos + 1 == self.verbatim then
-            -- check for raw attribute
-            local sp, ep =
-              bounded_find(subject, "^%{%=[^%s{}`]+%}", endchar + 1, endpos)
-            if sp and self.verbatim_type == "verbatim" then -- raw
-              self:add_match(pos, endchar, "-" .. self.verbatim_type)
-              self:add_match(sp, ep, "raw_format")
-              pos = ep + 1
-            else
-              self:add_match(pos, endchar, "-" .. self.verbatim_type)
-              pos = endchar + 1
-            end
-            self.verbatim = 0
-            self.verbatim_type = nil
-          else
-            endchar = endchar or endpos
-            self:add_match(pos, endchar, "str")
-            pos = endchar + 1
-          end
-        else
-          self:add_match(pos, pos, "str")
-          pos = pos + 1
-        end
-      else
-        pos = (matchers[c] and matchers[c](self, pos, endpos))
-               or self:single_char(pos)
-      end
-    end
-  end
-end
-
-  -- Return true if we're parsing verbatim content.
-function Parser:in_verbatim()
-  return self.verbatim > 0
-end
-
--- Return parse results and any warnings.
-function Parser:get_matches()
-  local sorted = {}
-  local subject = self.subject
-  local lastsp, lastep, lastannot
-  for i=self.firstpos, self.lastpos do
-    if self.matches[i] then
-      local sp, ep, annot = unpack_match(self.matches[i])
-      if annot == "str" and lastannot == "str" and lastep + 1 == sp then
-          -- consolidate adjacent strs
-        sorted[#sorted] = make_match(lastsp, ep, annot)
-        lastsp, lastep, lastannot = lastsp, ep, annot
-      else
-        sorted[#sorted + 1] = self.matches[i]
-        lastsp, lastep, lastannot = sp, ep, annot
-      end
-    end
-  end
-  if #sorted > 0 then
-    local last = sorted[#sorted]
-    local startpos, endpos, annot = unpack_match(last)
-    -- remove final softbreak
-    if annot == "softbreak" then
-      sorted[#sorted] = nil
-      last = sorted[#sorted]
-      startpos, endpos, annot = unpack_match(last)
-    end
-    -- remove trailing spaces
-    if annot == "str" and byte(subject, endpos) == 32 then
-      while endpos > startpos and byte(subject, endpos) == 32 do
-        endpos = endpos - 1
-      end
-      sorted[#sorted] = make_match(startpos, endpos, annot)
-    end
-    if self.verbatim > 0 then -- unclosed verbatim
-      self.warnings[#self.warnings + 1] =
-        {startpos, "Unclosed verbatim"}
-      sorted[#sorted + 1] = make_match(startpos, endpos,
-                                       "-" .. self.verbatim_type)
-    end
-  end
-  return sorted, self.warnings
-end
-
-return { Parser = Parser }
-
-
---[[
-Copyright (C) 2022 John MacFarlane
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be included
-in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-]]