extra-parser 0.6.2 → 0.6.4

package/README.md

@@ -118,7 +118,7 @@ function consumeToken<Token extends IToken = IToken>(
 ): Token | Falsy
 ```
 
-#### matchAnyOf
+### matchAnyOf
 ```ts
 function matchAnyOf<
   Token extends IToken = IToken
@@ -129,7 +129,7 @@ function matchAnyOf<
 ): Promise<INodePatternMatch<Node> | Falsy>
 ```
 
-#### matchSequence
+### matchSequence
 ```ts
 function matchSequence<
   Sequence extends ReadonlyArray<Token | Node>
@@ -141,7 +141,7 @@ function matchSequence<
 ): Promise<MapSequenceToMatches<Sequence, Token, Node> | Falsy>
 ```
 
-#### matchRepetitions
+### matchRepetitions
 ```ts
 function matchRepetitions<
   Sequence extends ReadonlyArray<Token | Node>

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "extra-parser",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "description": "A functional parser toolkit",
   "keywords": [],
   "files": [
-    "lib"
+    "lib",
+    "src"
   ],
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -31,7 +32,6 @@
     }
   },
   "devDependencies": {
-    "@blackglory/jest-matchers": "^0.5.0",
     "@commitlint/cli": "^17.2.0",
     "@commitlint/config-conventional": "^17.2.0",
     "@types/jest": "^29.2.2",
@@ -51,6 +51,7 @@
   },
   "dependencies": {
     "@blackglory/prelude": "^0.1.8",
+    "hotypes": "^0.5.1",
     "iterable-operator": "^2.3.0"
   }
 }

package/src/consume-node.ts

@@ -0,0 +1,22 @@
+import { Falsy, isntFalsy } from '@blackglory/prelude'
+import { IToken, INode, INodePattern, INodePatternMatch } from './types'
+
+/**
+ * 尝试匹配node, 如果成功, 则消耗掉相应的token.
+ * 
+ * @param tokens 匹配成功时会发生原地修改
+ */
+export async function consumeNode<
+  Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  nodePattern: INodePattern<Token, Node>
+, tokens: Token[]
+): Promise<INodePatternMatch<Node> | Falsy> {
+  const match = await nodePattern(tokens)
+
+  if (isntFalsy(match)) {
+    tokens.splice(0, match.consumed)
+    return match
+  }
+}

package/src/consume-token.ts

@@ -0,0 +1,19 @@
+import { Falsy } from '@blackglory/prelude'
+import { IToken } from './types'
+
+/**
+ * 尝试匹配token, 如果成功, 则消耗掉相应的token.
+ * 
+ * @param tokens 匹配成功时会发生原地修改
+ */
+export function consumeToken<Token extends IToken = IToken>(
+  tokenType: string
+, tokens: Token[]
+): Token | Falsy {
+  const firstToken: IToken | undefined = tokens[0]
+  
+  if (firstToken && firstToken.tokenType === tokenType) {
+    tokens.shift()
+    return firstToken as Token
+  }
+}

package/src/create-binary-operator-expression-node-pattern.ts

@@ -0,0 +1,50 @@
+import { isntFalsy } from '@blackglory/prelude'
+import { IToken, INode, INodePattern } from './types'
+import { matchSequence } from './match-sequence'
+
+export interface IBinaryOperatorExpressionNode<
+  NodeType extends string
+, LeftNode extends INode
+, RightNode extends INode
+> extends INode {
+  nodeType: NodeType
+  left: LeftNode
+  right: RightNode
+}
+
+export function createBinaryOperatorExpressionNodePattern<
+  Token extends IToken
+, Node extends IBinaryOperatorExpressionNode<string, LeftNode, RightNode>
+, LeftNode extends INode
+, RightNode extends INode
+>({ nodeType, centerTokenType, rightNodePattern, leftNodePattern }: {
+  nodeType: Node['nodeType']
+  centerTokenType: string
+  leftNodePattern: INodePattern<Token, LeftNode>
+  rightNodePattern: INodePattern<Token, RightNode>
+}): INodePattern<
+  Token
+, IBinaryOperatorExpressionNode<Node['nodeType'], Node['left'], Node['right']>
+> {
+  return async tokens => {
+    const matches = await matchSequence<[INode, IToken, INode]>(
+      [
+        leftNodePattern as INodePattern<IToken, LeftNode>
+      , centerTokenType
+      , rightNodePattern as INodePattern<IToken, RightNode>
+      ]
+    , tokens
+    )
+    if (isntFalsy(matches)) {
+      const [leftMatch, token, rightMatch] = matches
+      return {
+        consumed: leftMatch.consumed + 1 + rightMatch.consumed
+      , node: {
+          nodeType
+        , left: leftMatch.node as Node['left']
+        , right: rightMatch.node as Node['right']
+        }
+      }
+    }
+  }
+}

package/src/create-grouped-expression-node-pattern.ts

@@ -0,0 +1,30 @@
+import { isntFalsy } from '@blackglory/prelude'
+import { IToken, INode, INodePattern } from './types'
+import { matchSequence } from './match-sequence'
+
+export function createGroupedExpressionNodePattern<
+  Token extends IToken
+, CenterNode extends INode
+>({ leftTokenType, rightTokenType, centerNodePattern }: {
+  leftTokenType: string
+  rightTokenType: string
+  centerNodePattern: INodePattern<Token, CenterNode>
+}): INodePattern<Token, CenterNode> {
+  return async tokens => {
+    const matches = await matchSequence<[IToken, INode, IToken]>(
+      [
+        leftTokenType
+      , centerNodePattern as INodePattern<IToken, CenterNode>
+      , rightTokenType
+      ]
+    , tokens
+    )
+    if (isntFalsy(matches)) {
+      const [leftToken, nodeMatch, rightToken] = matches
+      return {
+        consumed: 1 + nodeMatch.consumed + 1
+      , node: nodeMatch.node as CenterNode
+      }
+    }
+  }
+}

package/src/create-token-pattern-from-regexp.ts

@@ -0,0 +1,32 @@
+import { Falsy, isntNull } from '@blackglory/prelude'
+import { IToken, ITokenPattern, ITokenPatternMatch } from './types'
+
+export function createTokenPatternFromRegExp<Token extends IToken>(
+  tokenType: Token['tokenType']
+, regExp: RegExp
+): ITokenPattern<IToken> {
+  const startsWithRegExp = convertToStartsWithRegExp(regExp)
+
+  return (text: string): ITokenPatternMatch<IToken> | Falsy => {
+    const result = startsWithRegExp.exec(text)
+    if (isntNull(result)) {
+      const [matchedText] = result
+      return {
+        consumed: matchedText.length
+      , token: {
+          tokenType: tokenType
+        , value: matchedText
+        }
+      }
+    } else {
+      return false
+    }
+  }
+}
+
+function convertToStartsWithRegExp(re: RegExp): RegExp {
+  return new RegExp(
+    re.source.startsWith('^') ? re.source : `^${re.source}`
+  , re.flags
+  )
+}

package/src/create-unary-operator-expression-node-pattern.ts

@@ -0,0 +1,44 @@
+import { isntFalsy } from '@blackglory/prelude'
+import { IToken, INode, INodePattern } from './types'
+import { matchSequence } from './match-sequence'
+
+export interface IUnaryOperatorExpressionNode<
+  NodeType extends string
+, RightNode extends INode
+> extends INode {
+  nodeType: NodeType
+  right: RightNode
+}
+
+export function createUnaryOperatorExpressionNodePattern<
+  Token extends IToken
+, Node extends IUnaryOperatorExpressionNode<string, RightNode>
+, RightNode extends INode
+>({ leftTokenType, nodeType, rightNodePattern }: {
+  nodeType: Node['nodeType']
+  leftTokenType: string
+  rightNodePattern: INodePattern<Token, RightNode>
+}): INodePattern<
+  Token
+, IUnaryOperatorExpressionNode<Node['nodeType'], Node['right']>
+> {
+  return async tokens => {
+    const matches = await matchSequence<[IToken, INode]>(
+      [
+        leftTokenType
+      , rightNodePattern as INodePattern<IToken, RightNode>
+      ]
+    , tokens
+    )
+    if (isntFalsy(matches)) {
+      const [leftToken, rightMatch] = matches
+      return {
+        consumed: 1 + rightMatch.consumed
+      , node: {
+          nodeType: nodeType
+        , right: rightMatch.node as RightNode
+        }
+      }
+    }
+  }
+}

package/src/create-value-expression-node-pattern.ts

@@ -0,0 +1,36 @@
+import { isntFalsy, toArray } from '@blackglory/prelude'
+import { consumeToken } from './consume-token'
+import { IToken, INode, INodePattern } from './types'
+
+export interface IValueExpressionNode<
+  NodeType extends string
+, Value
+> extends INode {
+  nodeType: NodeType
+  value: Value
+}
+
+export function createValueExpressionNodePattern<
+  Token extends IToken
+, Node extends IValueExpressionNode<string, Value>
+, Value
+>({ valueTokenType, nodeType, transformValue }: {
+  nodeType: Node['nodeType']
+  valueTokenType: string
+  transformValue: (value: string) => Value
+}): INodePattern<Token, IValueExpressionNode<Node['nodeType'], Node['value']>> {
+  return tokens => {
+    const mutableTokens = toArray(tokens)
+
+    const token = consumeToken(valueTokenType, mutableTokens)
+    if (isntFalsy(token)) {
+      return {
+        consumed: 1
+      , node: {
+          nodeType
+        , value: transformValue(token.value)
+        }
+      }
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,20 @@
+export {
+  IToken
+, ITokenPattern
+, ITokenPatternMatch
+, INode
+, INodePattern
+, INodePatternMatch
+} from './types'
+export * from './tokenize'
+export * from './parse'
+export * from './consume-token'
+export * from './consume-node'
+export { matchSequence } from './match-sequence'
+export * from './match-repetitions'
+export * from './match-any-of'
+export * from './create-token-pattern-from-regexp'
+export * from './create-unary-operator-expression-node-pattern'
+export * from './create-binary-operator-expression-node-pattern'
+export * from './create-grouped-expression-node-pattern'
+export * from './create-value-expression-node-pattern'

package/src/match-any-of.ts

@@ -0,0 +1,20 @@
+import { Falsy, isntFalsy } from '@blackglory/prelude'
+import { IToken, INode, INodePattern, INodePatternMatch } from './types'
+
+/**
+ * 从多个模式中依序匹配, 直到有一个匹配结果为真值, 返回该真值.
+ */
+export async function matchAnyOf<
+  Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  nodePatterns: ReadonlyArray<INodePattern<Token, Node>>
+, tokens: ReadonlyArray<Token>
+): Promise<INodePatternMatch<Node> | Falsy> {
+  for (const pattern of nodePatterns) {
+    const match = await pattern(tokens)
+    if (isntFalsy(match)) {
+      return match
+    }
+  }
+}

package/src/match-repetitions.ts

@@ -0,0 +1,59 @@
+import { assert, Falsy, isntFalsy, toArray } from '@blackglory/prelude'
+import { Flatten } from 'hotypes'
+import { IToken, INode, MapSequenceToPatterns, MapSequenceToMatches } from './types'
+import { matchSequence } from './match-sequence'
+
+export async function matchRepetitions<
+  Sequence extends ReadonlyArray<Token | Node>
+, Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  patterns: MapSequenceToPatterns<Sequence, Token, Node>
+, tokens: ReadonlyArray<Token>
+, {
+    minimumRepetitions = 1
+  , maximumRepetitions = Infinity
+  }: {
+    minimumRepetitions?: number
+    maximumRepetitions?: number
+  } = {}
+): Promise<Flatten<Array<MapSequenceToMatches<Sequence, Token, Node>>> | Falsy> {
+  assert(Number.isInteger(minimumRepetitions), 'The minimum repetiions must be an integer')
+  assert(
+    minimumRepetitions >= 0
+  , 'The minimum repetitions must be greater than or equal to 0'
+  )
+  assert(
+    Number.isInteger(maximumRepetitions) || maximumRepetitions === Infinity
+  , 'The maxmium repetiions must be an integer or an Infinity'
+  )
+  assert(
+    maximumRepetitions >= minimumRepetitions
+  , 'The maximum repetitions must be greater than or equal to the minimum repetitions'
+  )
+
+  const results: Array<MapSequenceToMatches<Sequence, Token, Node>> = []
+  const mutableTokens = toArray(tokens)
+
+  for (let i = 0; i < minimumRepetitions; i++) {
+    const matches = await matchSequence(patterns, mutableTokens)
+    if (isntFalsy(matches)) {
+      results.push(matches)
+      mutableTokens.splice(0, matches.length)
+    } else {
+      return
+    }
+  }
+
+  for (let i = minimumRepetitions; i < maximumRepetitions; i++) {
+    const matches = await matchSequence(patterns, mutableTokens)
+    if (isntFalsy(matches)) {
+      results.push(matches)
+      mutableTokens.splice(0, matches.length)
+    } else {
+      break
+    }
+  }
+
+  return results.flat() as Flatten<Array<MapSequenceToMatches<Sequence, Token, Node>>>
+}

package/src/match-sequence.ts

@@ -0,0 +1,180 @@
+import {
+  NonEmptyArray
+, Falsy
+, isntFalsy
+, isString
+, isFunction
+, toArray
+} from '@blackglory/prelude'
+import { findAllIndexes } from 'iterable-operator'
+import {
+  IToken
+, INode
+, INodePattern
+, INodePatternMatch
+, MapSequenceToPatterns
+, MapSequenceToMatches
+} from './types'
+import { consumeToken } from './consume-token'
+import { consumeNode } from './consume-node'
+
+/**
+ * 模式将被拆分为以下子模式来处理:
+ * - `[TokenType, ...TokenType[]]`
+ * - `[NodePattern, ...NodePattern[]]`:
+ *   根据NodePattern的定义, 这种子模式有可能陷入死循环.
+ * - `[NodePattern, TokenType]`:
+ *    这种子模式适用于二元或三元运算符这样的规则.
+ *    在引擎盖下, 它首先匹配TokenType以防止NodePattern在匹配时陷入死循环.
+ */
+export async function matchSequence<
+  Sequence extends ReadonlyArray<Token | Node>
+, Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  patterns: MapSequenceToPatterns<Sequence, Token, Node>
+, tokens: ReadonlyArray<Token>
+): Promise<MapSequenceToMatches<Sequence, Token, Node> | Falsy> {
+  if (isTokenTypes(patterns)) {
+    const matches: Array<Token> = []
+
+    const mutableTokens = toArray(tokens)
+    for (const pattern of patterns) {
+      const match = consumeToken(pattern, mutableTokens)
+      if (isntFalsy(match)) {
+        matches.push(match)
+      } else {
+        return
+      }
+    }
+
+    return matches as MapSequenceToMatches<Sequence, Token, Node>
+  } else if (isNodePatterns<Token, Node>(patterns)) {
+    const matches: Array<INodePatternMatch<Node>> = []
+
+    const mutableTokens = toArray(tokens)
+    for (const pattern of patterns) {
+      const match = await consumeNode(pattern, mutableTokens)
+      if (isntFalsy(match)) {
+        matches.push(match)
+      } else {
+        return
+      }
+    }
+
+    return matches as MapSequenceToMatches<Sequence, Token, Node>
+  } else if (isNodePatternNodeType<Token, Node>(patterns)) {
+    const [nodePattern, tokenType] = patterns
+
+    for (
+      const indexOfToken of findAllIndexes(tokens, x => x.tokenType === tokenType)
+    ) {
+      const leftTokens = tokens.slice(0, indexOfToken)
+      const leftMatch = await nodePattern(leftTokens)
+      if (
+        isntFalsy(leftMatch) &&
+        leftMatch.consumed === indexOfToken
+      ) {
+        const matches: [INodePatternMatch<Node>, Token] = [
+          leftMatch
+        , tokens[indexOfToken]
+        ]
+        return matches as MapSequenceToMatches<Sequence>
+      }
+    }
+  } else {
+    const matches: Array<INodePatternMatch<Node> | Token> = []
+    const remainingTokens = toArray(tokens)
+    for (const subPatterns of splitPatterns(patterns)) {
+      const subMatches = await matchSequence(
+        subPatterns as MapSequenceToPatterns<Sequence, Token, Node>
+      , remainingTokens
+      )
+      if (isntFalsy(subMatches)) {
+        const consumed = subMatches
+          .map((match: Token | INodePatternMatch<Node>) => {
+            return 'consumed' in match
+                 ? match.consumed
+                 : 1
+          })
+          .reduce((acc, cur) => acc + cur, 0)
+        remainingTokens.splice(0, consumed)
+        matches.push(...subMatches)
+      } else {
+        return
+      }
+    }
+    return matches as MapSequenceToMatches<Sequence>
+  }
+}
+
+type SubPatterns<Token extends IToken = IToken, Node extends INode = INode> =
+| [INodePattern<Token, Node>, string]
+| NonEmptyArray<string>
+| NonEmptyArray<INodePattern<Token, Node>>
+
+/**
+ * 该函数会匹配尽可能长的subPatterns.
+ */
+export function* splitPatterns<
+  Sequence extends ReadonlyArray<Token | Node>
+, Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  patterns: MapSequenceToPatterns<Sequence, Token, Node>
+): IterableIterator<SubPatterns<Token, Node>> {
+  const mutablePatterns: Array<INodePattern<Token, Node> | string> = toArray(patterns)
+
+  while (mutablePatterns.length > 0) {
+    if (isTokenType(mutablePatterns[0])) {
+      const indexOfNodePattern = mutablePatterns.findIndex(x => isNodePattern(x))
+      if (indexOfNodePattern === -1) {
+        yield mutablePatterns.splice(0) as NonEmptyArray<string>
+      } else {
+        yield mutablePatterns.splice(0, indexOfNodePattern) as NonEmptyArray<string>
+      }
+    } else if (isNodePattern(mutablePatterns[0])) {
+      const indexOfToken = mutablePatterns.findIndex(x => isTokenType(x))
+      if (indexOfToken === -1) {
+        yield mutablePatterns.splice(0) as NonEmptyArray<
+          INodePattern<Token, Node>
+        >
+      } else {
+        yield mutablePatterns.splice(0, indexOfToken + 1) as [
+          INodePattern<Token, Node>
+        , string
+        ]
+      }
+    } else {
+      throw new Error('Unknown patterns')
+    }
+  }
+}
+
+function isTokenTypes(arr: ReadonlyArray<unknown>): arr is ReadonlyArray<string> {
+  return arr.every(isTokenType)
+}
+
+function isTokenType(val: unknown): val is string {
+  return isString(val)
+}
+
+function isNodePatterns<Token extends IToken = IToken, Node extends INode = INode>(
+  arr: ReadonlyArray<unknown>
+): arr is ReadonlyArray<INodePattern<Token, Node>> {
+  return arr.every(isNodePattern)
+}
+
+function isNodePattern<Token extends IToken = IToken, Node extends INode = INode>(
+  val: unknown
+): val is INodePattern<Token, Node> {
+  return isFunction(val)
+}
+
+function isNodePatternNodeType<Token extends IToken = IToken, Node extends INode = INode>(
+  arr: ReadonlyArray<unknown>
+): arr is readonly [INodePattern<Token, Node>, string] {
+  return arr.length === 2
+      && isNodePattern(arr[0])
+      && isTokenType(arr[1])
+}

package/src/parse.ts

@@ -0,0 +1,26 @@
+import { isntFalsy } from '@blackglory/prelude'
+import { IToken, INodePattern, INode } from './types'
+
+export async function* parse<
+  Token extends IToken = IToken
+, Node extends INode = INode
+>(
+  patterns: Array<INodePattern<Token, Node>>
+, tokens: Token[]
+): AsyncIterableIterator<Node> {
+  let i = 0
+  loop: while (i < tokens.length) {
+    const remainingTokens = tokens.slice(i)
+
+    for (const pattern of patterns) {
+      const result = await pattern(remainingTokens)
+      if (isntFalsy(result)) {
+        yield result.node
+        i += result.consumed
+        continue loop
+      }
+    }
+
+    throw new Error(`Unparseable tokens: ${JSON.stringify(remainingTokens)}`)
+  }
+}

package/src/tokenize.ts

@@ -0,0 +1,23 @@
+import { isntFalsy } from '@blackglory/prelude'
+import { ITokenPattern, IToken } from './types'
+
+export async function* tokenize<Token extends IToken = IToken>(
+  patterns: Array<ITokenPattern<Token>>
+, text: string
+): AsyncIterableIterator<Token> {
+  let i = 0
+  loop: while (i < text.length) {
+    const remainingText = text.slice(i)
+
+    for (const pattern of patterns) {
+      const result = await pattern(remainingText)
+      if (isntFalsy(result)) {
+        yield result.token
+        i += result.consumed
+        continue loop
+      }
+    }
+
+    throw new Error(`Unknown text: ${JSON.stringify(remainingText)}`)
+  }
+}

package/src/types.ts

@@ -0,0 +1,65 @@
+import { Falsy, Awaitable } from '@blackglory/prelude'
+
+export interface IToken {
+  tokenType: string
+  value: string
+}
+
+export interface INode {
+  nodeType: string
+}
+
+export interface ITokenPatternMatch<Token extends IToken> {
+  consumed: number
+  token: Token
+}
+
+export interface INodePatternMatch<Node extends INode> {
+  consumed: number
+  node: Node
+}
+
+export interface ITokenPattern<Token extends IToken = IToken> {
+  (text: string): Awaitable<ITokenPatternMatch<Token> | Falsy>
+}
+
+export interface INodePattern<
+  Token extends IToken = IToken
+, Node extends INode = INode
+> {
+  (tokens: ReadonlyArray<Token>): Awaitable<INodePatternMatch<Node> | Falsy>
+}
+
+export type MapSequenceToPatterns<
+  Sequence extends ReadonlyArray<Token | Node>
+, Token extends IToken = IToken
+, Node extends INode = INode
+> = {
+  [Index in keyof Sequence]:
+    [Sequence[Index]] extends [infer Element]
+  ? (
+      Element extends Token
+      ? string
+    : Element extends Node
+      ? INodePattern<Token, Element>
+    : never
+    )
+  : never
+}
+
+export type MapSequenceToMatches<
+  Sequence extends ReadonlyArray<Token | Node>
+, Token extends IToken = IToken
+, Node extends INode = INode
+> = {
+  [Index in keyof Sequence]:
+    [Sequence[Index]] extends [infer Element]
+  ? (
+      Element extends IToken
+      ? Token
+    : Element extends INode
+      ? INodePatternMatch<Element>
+    : never
+    )
+  : never
+}