@yozora/tokenizer-image 2.0.3 → 2.0.5-alpha.0

package/lib/cjs/{index.js → index.cjs}

@@ -142,19 +142,18 @@ const uniqueName = '@yozora/tokenizer-image';
 
 class ImageTokenizer extends coreTokenizer.BaseInlineTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : coreTokenizer.TokenizerPriority.LINKS,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? coreTokenizer.TokenizerPriority.LINKS,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 exports.ImageTokenizer = ImageTokenizer;
 exports.ImageTokenizerName = uniqueName;
 exports.calcImageAlt = calcImageAlt;
-exports["default"] = ImageTokenizer;
+exports.default = ImageTokenizer;
 exports.imageMatch = match;
 exports.imageParse = parse;

package/lib/esm/{index.js → index.mjs}

@@ -138,14 +138,13 @@ const uniqueName = '@yozora/tokenizer-image';
 
 class ImageTokenizer extends BaseInlineTokenizer {
     constructor(props = {}) {
-        var _a, _b;
         super({
-            name: (_a = props.name) !== null && _a !== void 0 ? _a : uniqueName,
-            priority: (_b = props.priority) !== null && _b !== void 0 ? _b : TokenizerPriority.LINKS,
+            name: props.name ?? uniqueName,
+            priority: props.priority ?? TokenizerPriority.LINKS,
         });
-        this.match = match;
-        this.parse = parse;
     }
+    match = match;
+    parse = parse;
 }
 
 export { ImageTokenizer, uniqueName as ImageTokenizerName, calcImageAlt, ImageTokenizer as default, match as imageMatch, parse as imageParse };

package/lib/types/index.d.ts

@@ -10,8 +10,8 @@ import { INodeInterval } from '@yozora/character';
  */
 declare function calcImageAlt(nodes: ReadonlyArray<Node>): string;
 
-declare type T = ImageType;
-declare type INode = Image;
+type T = ImageType;
+type INode = Image;
 declare const uniqueName = "@yozora/tokenizer-image";
 /**
  * An image token.
@@ -40,8 +40,8 @@ interface IDelimiter extends IYastTokenDelimiter {
      */
     titleContent?: INodeInterval;
 }
-declare type IThis = ITokenizer;
-declare type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
+type IThis = ITokenizer;
+type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;
 
 /**
  * Syntax for images is like the syntax for links, with one difference.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yozora/tokenizer-image",
-  "version": "2.0.3",
+  "version": "2.0.5-alpha.0",
   "author": {
     "name": "guanghechen",
     "url": "https://github.com/guanghechen/"
@@ -11,34 +11,38 @@
     "directory": "tokenizers/image"
   },
   "homepage": "https://github.com/yozorajs/yozora/tree/release-2.x.x/tokenizers/image",
-  "main": "lib/cjs/index.js",
-  "module": "lib/esm/index.js",
-  "types": "lib/types/index.d.ts",
-  "source": "src/index.ts",
+  "type": "module",
+  "exports": {
+    "types": "./lib/types/index.d.ts",
+    "import": "./lib/esm/index.mjs",
+    "require": "./lib/cjs/index.cjs"
+  },
+  "source": "./src/index.ts",
+  "types": "./lib/types/index.d.ts",
+  "main": "./lib/cjs/index.cjs",
+  "module": "./lib/esm/index.mjs",
   "license": "MIT",
   "engines": {
     "node": ">= 16.0.0"
   },
   "files": [
     "lib/",
-    "!lib/**/*.js.map",
-    "!lib/**/*.d.ts.map",
+    "src/",
     "package.json",
     "CHANGELOG.md",
     "LICENSE",
     "README.md"
   ],
   "scripts": {
-    "build": "cross-env NODE_ENV=production rollup -c ../../rollup.config.js",
-    "prebuild": "rimraf lib/",
+    "build": "rimraf lib/ && cross-env NODE_ENV=production rollup -c ../../rollup.config.mjs",
     "prepublishOnly": "cross-env ROLLUP_SHOULD_SOURCEMAP=false yarn build",
-    "test": "cross-env TS_NODE_FILES=true jest --config ../../jest.config.js --rootDir ."
+    "test": "cross-env TS_NODE_FILES=true NODE_OPTIONS=--experimental-vm-modules jest --config ../../jest.config.mjs --rootDir ."
   },
   "dependencies": {
-    "@yozora/ast": "^2.0.3",
-    "@yozora/character": "^2.0.3",
-    "@yozora/core-tokenizer": "^2.0.3",
-    "@yozora/tokenizer-link": "^2.0.3"
+    "@yozora/ast": "^2.0.5-alpha.0",
+    "@yozora/character": "^2.0.5-alpha.0",
+    "@yozora/core-tokenizer": "^2.0.5-alpha.0",
+    "@yozora/tokenizer-link": "^2.0.5-alpha.0"
   },
-  "gitHead": "8cc8f95cfebc8d752bc3272cdd24965f540c130b"
+  "gitHead": "8bf941fe4ef82947165b0f3cc123cd493665e13b"
 }

package/src/index.ts

@@ -0,0 +1,10 @@
+export * from './util'
+export { match as imageMatch } from './match'
+export { parse as imageParse } from './parse'
+export { ImageTokenizer, ImageTokenizer as default } from './tokenizer'
+export { uniqueName as ImageTokenizerName } from './types'
+export type {
+  IThis as IImageHookContext,
+  IToken as IImageToken,
+  ITokenizerProps as IImageTokenizerProps,
+} from './types'

package/src/match.ts

@@ -0,0 +1,211 @@
+import { ImageType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { AsciiCodePoint } from '@yozora/character'
+import type {
+  IMatchInlineHookCreator,
+  IResultOfIsDelimiterPair,
+  IResultOfProcessDelimiterPair,
+  IYastInlineToken,
+} from '@yozora/core-tokenizer'
+import { eatOptionalWhitespaces, genFindDelimiter } from '@yozora/core-tokenizer'
+import {
+  checkBalancedBracketsStatus,
+  eatLinkDestination,
+  eatLinkTitle,
+} from '@yozora/tokenizer-link'
+import type { IDelimiter, IThis, IToken, T } from './types'
+
+/**
+ * Syntax for images is like the syntax for links, with one difference.
+ * Instead of link text, we have an image description.
+ * The rules for this are the same as for link text, except that
+ *
+ *  a) an image description starts with '![' rather than '[', and
+ *  b) an image description may contain links.
+ *
+ * An image description has inline elements as its contents. When an image is
+ * rendered to HTML, this is standardly used as the image’s alt attribute.
+ *
+ * ------
+ *
+ * A 'opener' type delimiter is one of the following forms:
+ *
+ *  - '!['
+ *
+ * A 'closer' type delimiter is one of the following forms:
+ *
+ *  - '](url)'
+ *  - '](url "title")'
+ *  - '](<url>)'
+ *  - '](<url> "title")'
+ *
+ * @see https://github.com/syntax-tree/mdast#image
+ * @see https://github.github.com/gfm/#images
+ */
+export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
+  return {
+    findDelimiter: () => genFindDelimiter<IDelimiter>(_findDelimiter),
+    isDelimiterPair,
+    processDelimiterPair,
+  }
+  /**
+   * Images can contains Images, so implement an algorithm similar to bracket
+   * matching, pushing all opener delimiters onto the stack
+   *
+   * The rules for this are the same as for link text, except that
+   *  (a) an image description starts with '![' rather than '[', and
+   *  (b) an image description may contain links. An image description has
+   *      inline elements as its contents. When an image is rendered to HTML,
+   *      this is standardly used as the image’s alt attribute
+   *
+   * @see https://github.github.com/gfm/#inline-link
+   * @see https://github.github.com/gfm/#example-582
+   *
+   * @override
+   * @see IMatchInlineHook
+   */
+  function _findDelimiter(startIndex: number, endIndex: number): IDelimiter | null {
+    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+    const blockEndIndex = api.getBlockEndIndex()
+
+    /**
+     * FIXME:
+     *
+     * This is a hack method to fix the situation where a higher priority token
+     * is embedded in the delimiter, at this time, ignore the tokens that have
+     * been parsed, and continue to match the content until the delimiter meets
+     * its own definition or reaches the right boundary of the block content.
+     *
+     * This algorithm has not been strictly logically verified, but I think it
+     * can work well in most cases. After all, it has passed many test cases.
+     * @see https://github.github.com/gfm/#example-588
+     */
+    for (let i = startIndex; i < endIndex; ++i) {
+      const c = nodePoints[i].codePoint
+      switch (c) {
+        case AsciiCodePoint.BACKSLASH:
+          i += 1
+          break
+        case AsciiCodePoint.EXCLAMATION_MARK: {
+          if (i + 1 < endIndex && nodePoints[i + 1].codePoint === AsciiCodePoint.OPEN_BRACKET) {
+            return {
+              type: 'opener',
+              startIndex: i,
+              endIndex: i + 2,
+            }
+          }
+          break
+        }
+        /**
+         * An inline link consists of a link text followed immediately by a
+         * left parenthesis '(', ..., and a right parenthesis ')'
+         * @see https://github.github.com/gfm/#inline-link
+         */
+        case AsciiCodePoint.CLOSE_BRACKET: {
+          /**
+           * An inline link consists of a link text followed immediately by a
+           * left parenthesis '('
+           * @see https://github.github.com/gfm/#inline-link
+           */
+          if (
+            i + 1 >= endIndex ||
+            nodePoints[i + 1].codePoint !== AsciiCodePoint.OPEN_PARENTHESIS
+          ) {
+            break
+          }
+
+          // try to match link destination
+          const destinationStartIndex = eatOptionalWhitespaces(nodePoints, i + 2, blockEndIndex)
+          const destinationEndIndex = eatLinkDestination(
+            nodePoints,
+            destinationStartIndex,
+            blockEndIndex,
+          )
+          if (destinationEndIndex < 0) break
+
+          // try to match link title
+          const titleStartIndex = eatOptionalWhitespaces(
+            nodePoints,
+            destinationEndIndex,
+            blockEndIndex,
+          )
+          const titleEndIndex = eatLinkTitle(nodePoints, titleStartIndex, blockEndIndex)
+          if (titleEndIndex < 0) break
+
+          const _startIndex = i
+          const _endIndex = eatOptionalWhitespaces(nodePoints, titleEndIndex, blockEndIndex) + 1
+          if (
+            _endIndex > blockEndIndex ||
+            nodePoints[_endIndex - 1].codePoint !== AsciiCodePoint.CLOSE_PARENTHESIS
+          ) {
+            break
+          }
+
+          /**
+           * Both the title and the destination may be omitted
+           * @see https://github.github.com/gfm/#example-495
+           */
+          return {
+            type: 'closer',
+            startIndex: _startIndex,
+            endIndex: _endIndex,
+            destinationContent:
+              destinationStartIndex < destinationEndIndex
+                ? {
+                    startIndex: destinationStartIndex,
+                    endIndex: destinationEndIndex,
+                  }
+                : undefined,
+            titleContent:
+              titleStartIndex < titleEndIndex
+                ? { startIndex: titleStartIndex, endIndex: titleEndIndex }
+                : undefined,
+          }
+        }
+      }
+    }
+    return null
+  }
+
+  function isDelimiterPair(
+    openerDelimiter: IDelimiter,
+    closerDelimiter: IDelimiter,
+    internalTokens: ReadonlyArray<IYastInlineToken>,
+  ): IResultOfIsDelimiterPair {
+    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+    const balancedBracketsStatus: -1 | 0 | 1 = checkBalancedBracketsStatus(
+      openerDelimiter.endIndex,
+      closerDelimiter.startIndex,
+      internalTokens,
+      nodePoints,
+    )
+    switch (balancedBracketsStatus) {
+      case -1:
+        return { paired: false, opener: false, closer: true }
+      case 0:
+        return { paired: true }
+      case 1:
+        return { paired: false, opener: true, closer: false }
+    }
+  }
+
+  function processDelimiterPair(
+    openerDelimiter: IDelimiter,
+    closerDelimiter: IDelimiter,
+    internalTokens: ReadonlyArray<IYastInlineToken>,
+  ): IResultOfProcessDelimiterPair<T, IToken, IDelimiter> {
+    const token: IToken = {
+      nodeType: ImageType,
+      startIndex: openerDelimiter.startIndex,
+      endIndex: closerDelimiter.endIndex,
+      destinationContent: closerDelimiter.destinationContent,
+      titleContent: closerDelimiter.titleContent,
+      children: api.resolveInternalTokens(
+        internalTokens,
+        openerDelimiter.endIndex,
+        closerDelimiter.startIndex,
+      ),
+    }
+    return { tokens: [token] }
+  }
+}

package/src/parse.ts

@@ -0,0 +1,50 @@
+import type { Node } from '@yozora/ast'
+import { ImageType } from '@yozora/ast'
+import type { INodePoint } from '@yozora/character'
+import { AsciiCodePoint, calcEscapedStringFromNodePoints } from '@yozora/character'
+import type { IParseInlineHookCreator } from '@yozora/core-tokenizer'
+import { encodeLinkDestination } from '@yozora/core-tokenizer'
+import type { INode, IThis, IToken, T } from './types'
+import { calcImageAlt } from './util'
+
+export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: tokens =>
+      tokens.map(token => {
+        const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints()
+
+        // calc url
+        let url = ''
+        if (token.destinationContent != null) {
+          let { startIndex, endIndex } = token.destinationContent
+          if (nodePoints[startIndex].codePoint === AsciiCodePoint.OPEN_ANGLE) {
+            startIndex += 1
+            endIndex -= 1
+          }
+          const destination = calcEscapedStringFromNodePoints(
+            nodePoints,
+            startIndex,
+            endIndex,
+            true,
+          )
+          url = encodeLinkDestination(destination)
+        }
+
+        // calc alt
+        const children: Node[] = api.parseInlineTokens(token.children)
+        const alt = calcImageAlt(children)
+
+        // calc title
+        let title: string | undefined
+        if (token.titleContent != null) {
+          const { startIndex, endIndex } = token.titleContent
+          title = calcEscapedStringFromNodePoints(nodePoints, startIndex + 1, endIndex - 1)
+        }
+
+        const node: INode = api.shouldReservePosition
+          ? { type: ImageType, position: api.calcPosition(token), url, alt, title }
+          : { type: ImageType, url, alt, title }
+        return node
+      }),
+  }
+}

package/src/tokenizer.ts

@@ -0,0 +1,32 @@
+import type {
+  IInlineTokenizer,
+  IMatchInlineHookCreator,
+  IParseInlineHookCreator,
+} from '@yozora/core-tokenizer'
+import { BaseInlineTokenizer, TokenizerPriority } from '@yozora/core-tokenizer'
+import { match } from './match'
+import { parse } from './parse'
+import type { IDelimiter, INode, IThis, IToken, ITokenizerProps, T } from './types'
+import { uniqueName } from './types'
+
+/**
+ * Lexical Analyzer for InlineImage.
+ * @see https://github.com/syntax-tree/mdast#image
+ * @see https://github.github.com/gfm/#images
+ */
+export class ImageTokenizer
+  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+{
+  /* istanbul ignore next */
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: props.name ?? uniqueName,
+      priority: props.priority ?? TokenizerPriority.LINKS,
+    })
+  }
+
+  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match
+
+  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse
+}

package/src/types.ts

@@ -0,0 +1,45 @@
+import type { Image, ImageType } from '@yozora/ast'
+import type { INodeInterval } from '@yozora/character'
+import type {
+  IBaseInlineTokenizerProps,
+  IPartialYastInlineToken,
+  ITokenizer,
+  IYastTokenDelimiter,
+} from '@yozora/core-tokenizer'
+
+export type T = ImageType
+export type INode = Image
+export const uniqueName = '@yozora/tokenizer-image'
+
+/**
+ * An image token.
+ */
+export interface IToken extends IPartialYastInlineToken<T> {
+  /**
+   * Link destination interval.
+   */
+  destinationContent?: INodeInterval
+  /**
+   * Link title interval.
+   */
+  titleContent?: INodeInterval
+}
+
+export interface IDelimiter extends IYastTokenDelimiter {
+  /**
+   * IDelimiter type.
+   */
+  type: 'opener' | 'closer'
+  /**
+   * link destination
+   */
+  destinationContent?: INodeInterval
+  /**
+   * link title
+   */
+  titleContent?: INodeInterval
+}
+
+export type IThis = ITokenizer
+
+export type ITokenizerProps = Partial<IBaseInlineTokenizerProps>

package/src/util.ts

@@ -0,0 +1,18 @@
+import type { Alternative, Literal, Node, Parent } from '@yozora/ast'
+
+/**
+ * calc alt
+ * An image description has inline elements as its contents. When an image
+ * is rendered to HTML, this is standardly used as the image’s alt attribute
+ * @see https://github.github.com/gfm/#example-582
+ */
+export function calcImageAlt(nodes: ReadonlyArray<Node>): string {
+  return (nodes as ReadonlyArray<Node & Alternative & Literal & Parent>)
+    .map((o): string => {
+      if (o.value != null) return o.value
+      if (o.alt != null) return o.alt
+      if (o.children != null) return calcImageAlt(o.children)
+      return ''
+    })
+    .join('')
+}