@flex-development/docmark 1.0.0-dev.3

package/LICENSE.md

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Flex Development, LLC All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/README.md

@@ -0,0 +1,92 @@
+# docmark
+
+[![npm](https://img.shields.io/npm/v/@flex-development/docmark.svg)](https://npmjs.com/package/@flex-development/docmark)
+[![npm downloads](https://img.shields.io/npm/dm/@flex-development/docmark.svg)](https://www.npmcharts.com/compare/@flex-development/docmark?interval=30)
+[![install size](https://packagephobia.now.sh/badge?p=@flex-development/docmark)](https://packagephobia.now.sh/result?p=@flex-development/docmark)
+[![module type: esm](https://img.shields.io/badge/module%20type-esm-brightgreen)](https://github.com/voxpelli/badges-cjs-esm)
+
+a comment parser with support for markdown.
+
+> **Note**: This is the `docmark` package from the docmark monorepo.
+> See the [monorepo readme][docmark] for more on the project.
+> See this readme for how to use it.
+
+## Contents
+
+- [What is this?](#what-is-this)
+- [When should I use this?](#when-should-i-use-this)
+- [Install](#install)
+- [Use](#use)
+- [API](#api)
+- [Types](#types)
+- [Contribute](#contribute)
+
+## What is this?
+
+**TODO**: what is this?
+
+## When should I use this?
+
+docmark parses documentation comments, not just docblocks.
+Extensions can register additional comment syntaxes — such as line comments, hash comments, or language-specific
+documentation comments — without changing how the comment content itself is parsed.
+
+## Install
+
+This package is [ESM only][esm].
+
+In Node.js (version 20+) with [yarn][]:
+
+```sh
+yarn add @flex-development/docmark
+```
+
+<blockquote>
+  <small>
+    See <a href='https://yarnpkg.com/protocol/git'>Git - Protocols | Yarn</a>
+    &nbsp;for details regarding installing from Git.
+  </small>
+</blockquote>
+
+In Deno with [`esm.sh`][esmsh]:
+
+```ts
+import { docmark } from 'https://esm.sh/@flex-development/docmark'
+```
+
+In browsers with [`esm.sh`][esmsh]:
+
+```html
+<script type="module">
+  import { docmark } from 'https://esm.sh/@flex-development/docmark'
+</script>
+```
+
+## Use
+
+**TODO**: use
+
+## API
+
+**TODO**: api
+
+## Types
+
+This package is fully typed with [TypeScript][].
+
+## Contribute
+
+See [`CONTRIBUTING.md`](../../CONTRIBUTING.md).
+
+This project has a [code of conduct](../../CODE_OF_CONDUCT.md).
+By interacting with this repository, organization, or community you agree to abide by its terms.
+
+[docmark]: ../../README.md
+
+[esm]: https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c
+
+[esmsh]: https://esm.sh
+
+[typescript]: https://www.typescriptlang.org
+
+[yarn]: https://yarnpkg.com

package/dist/index.d.mts

@@ -0,0 +1,67 @@
+import type { ParseOptions, ParseContext, Event, PreprocessOptions, Preprocessor } from '@flex-development/docmark-util-types';
+
+/**
+ * @file parse
+ * @module docmark/parse
+ */
+
+/**
+ * Create a parser.
+ *
+ * Tokenizers deal with one content type.
+ * The parser is the object dealing with it all.
+ *
+ * @see {@linkcode ParseContext}
+ * @see {@linkcode ParseOptions}
+ *
+ * @this {void}
+ *
+ * @param {ParseOptions | null | undefined} [options]
+ *  Options for parsing
+ * @return {ParseContext}
+ *  The parse context
+ */
+declare function parse(this: void, options?: ParseOptions | null | undefined): ParseContext;
+
+/**
+ * @file postprocess
+ * @module docmark/postprocess
+ */
+
+/**
+ * Postprocess events.
+ *
+ * @todo resolve trailing whitespace
+ *
+ * @see {@linkcode Event}
+ *
+ * @this {void}
+ *
+ * @param {Event[]} events
+ *  The current list of events
+ * @return {Event[]}
+ *  The list of changed events
+ */
+declare function postprocess(this: void, events: Event[]): Event[];
+
+/**
+ * @file preprocess
+ * @module docmark/preprocess
+ */
+
+/**
+ * Create a preprocessor to turn a value into chunks.
+ *
+ * @see {@linkcode Preprocessor}
+ * @see {@linkcode PreprocessOptions}
+ *
+ * @this {void}
+ *
+ * @param {PreprocessOptions | null | undefined} [options]
+ *  The configuration options
+ * @return {Preprocessor}
+ *  The preprocessor
+ */
+declare function preprocess(this: void, options?: PreprocessOptions | null | undefined): Preprocessor;
+
+export { parse, postprocess, preprocess };

package/dist/index.mjs

@@ -0,0 +1,1121 @@
+import { combineExtensions } from '@flex-development/docmark-util-combine-extensions';
+import { ct, tt, codes, constants, ev } from '@flex-development/docmark-util-symbol';
+import { createTokenizer } from '@flex-development/mark-parser';
+import { eos, bos, eol, whitespace, htab } from '@flex-development/mark-util-character';
+import { typeExpressionValue, summary, blankLine, region } from '@flex-development/docmark-grammar';
+import * as commonmark from 'micromark-core-commonmark';
+import { splice } from '@flex-development/mark-util-chunked';
+import { ok } from 'devlop';
+import { subtokenize } from '@flex-development/docmark-util-subtokenize';
+import { factorySpace } from '@flex-development/docmark-factory-space';
+import { subtokenize as subtokenize$1 } from 'micromark-util-subtokenize';
+import { decode } from '@flex-development/mark-parser/utils';
+
+const docmark = {
+    comment: { null: summary },
+    source: {},
+    type: { null: typeExpressionValue }
+};
+
+const string = initializeFactory(ct.string);
+const text = initializeFactory(ct.text);
+function initializeFactory(contentType) {
+    const capt = `${contentType[0].toUpperCase()}${contentType.slice(1)}`;
+    const tokenizer = tokenize.name + capt;
+    return {
+        resolveAll: resolver(contentType === 'text' && resolveLineSuffixes),
+        tokenize: Object.defineProperties(tokenize, { name: { value: tokenizer } })
+    };
+    function tokenize(effects) {
+        const self = this;
+        const constructs = self.parser.constructs[contentType];
+        const state = effects.attempt(constructs, start, notText);
+        return start;
+        function start(code) {
+            return atBreak(code) ? state(code) : notText(code);
+        }
+        function notText(code) {
+            if (eos(code))
+                return void effects.consume(code);
+            return effects.enter(tt.data), effects.consume(code), data;
+        }
+        function data(code) {
+            if (atBreak(code))
+                return effects.exit(tt.data), state(code);
+            return effects.consume(code), data;
+        }
+        function atBreak(code) {
+            if (eos(code))
+                return true;
+            const pack = constructs[code];
+            if (pack) {
+                ok(Array.isArray(pack), 'expected list of constructs');
+                let index = -1;
+                while (++index < pack.length) {
+                    ok(pack[index], 'expected `pack[index]`');
+                    const item = pack[index];
+                    if (!item.previous || item.previous.call(self, self.previous)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+    }
+    function resolver(extraResolver) {
+        const name = resolveAll.name + capt;
+        Object.defineProperties(resolveAll, { name: { value: name } });
+        return resolveAll;
+        function resolveAll(events, context) {
+            let index = -1;
+            let enter = undefined;
+            while (++index <= events.length) {
+                if (enter === undefined) {
+                    if (events[index] && events[index][1].type === tt.data) {
+                        enter = index;
+                        index++;
+                    }
+                }
+                else if (!events[index] || events[index][1].type !== tt.data) {
+                    if (index !== enter + 2) {
+                        events[enter][1].end = events[index - 1][1].end;
+                        events.splice(enter + 2, index - enter - 2);
+                        index = enter + 2;
+                    }
+                    enter = undefined;
+                }
+            }
+            return extraResolver ? extraResolver(events, context) : events;
+        }
+    }
+}
+function resolveLineSuffixes(events) {
+    let index = 0;
+    while (++index <= events.length) {
+        if (index === events.length || events[index][1].type === tt.lineEnding) {
+            if (events[index - 1][1].type === tt.data) {
+                const [, data, context] = events[index - 1];
+                const chunks = context.sliceStream(data);
+                let bufferIndex = -1;
+                let chunkIndex = chunks.length;
+                let size = 0;
+                let tabs = undefined;
+                while (chunkIndex--) {
+                    const chunk = chunks[chunkIndex];
+                    if (typeof chunk === 'string') {
+                        bufferIndex = chunk.length;
+                        while (chunk.codePointAt(bufferIndex - 1) === codes.space) {
+                            size++;
+                            bufferIndex--;
+                        }
+                        if (bufferIndex)
+                            break;
+                        bufferIndex = -1;
+                    }
+                    else if (chunk === codes.horizontalTab) {
+                        tabs = true;
+                        size++;
+                    }
+                    else if (chunk !== codes.virtualSpace) {
+                        chunkIndex++;
+                        break;
+                    }
+                }
+                if (size) {
+                    const _bufferIndex = chunkIndex
+                        ? bufferIndex
+                        : data.start._bufferIndex + bufferIndex;
+                    const lineSuffix = index === events.length ||
+                        tabs ||
+                        size < constants.hardBreakPrefixSizeMin;
+                    const start = {
+                        line: data.end.line,
+                        column: data.end.column - size,
+                        offset: data.end.offset - size,
+                        _index: data.start._index + chunkIndex,
+                        _bufferIndex
+                    };
+                    const type = lineSuffix
+                        ? tt.lineSuffix
+                        : tt.hardBreakTrailing;
+                    const token = context.token(type, {
+                        end: structuredClone(data.end),
+                        start: (data.end = structuredClone(start), start)
+                    });
+                    if (data.start.offset === data.end.offset) {
+                        Object.assign(data, token);
+                    }
+                    else {
+                        const enter = [ev.enter, token, context];
+                        const exit = [ev.exit, token, context];
+                        splice(events, index, 0, [enter, exit]);
+                        index += 2;
+                    }
+                }
+                index++;
+            }
+        }
+    }
+    return events;
+}
+
+const markdown = {
+    attentionMarkers: {
+        null: [codes.asterisk, codes.underscore]
+    },
+    contentInitial: {
+        [codes.leftSquareBracket]: commonmark.definition
+    },
+    disable: {
+        null: []
+    },
+    document: {
+        [codes.asterisk]: commonmark.list,
+        [codes.plusSign]: commonmark.list,
+        [codes.dash]: commonmark.list,
+        [codes.digit0]: commonmark.list,
+        [codes.digit1]: commonmark.list,
+        [codes.digit2]: commonmark.list,
+        [codes.digit3]: commonmark.list,
+        [codes.digit4]: commonmark.list,
+        [codes.digit5]: commonmark.list,
+        [codes.digit6]: commonmark.list,
+        [codes.digit7]: commonmark.list,
+        [codes.digit8]: commonmark.list,
+        [codes.digit9]: commonmark.list,
+        [codes.greaterThan]: commonmark.blockQuote
+    },
+    flow: {
+        [codes.numberSign]: commonmark.headingAtx,
+        [codes.asterisk]: commonmark.thematicBreak,
+        [codes.dash]: [commonmark.setextUnderline, commonmark.thematicBreak],
+        [codes.lessThan]: commonmark.htmlFlow,
+        [codes.equalsTo]: commonmark.setextUnderline,
+        [codes.underscore]: commonmark.thematicBreak,
+        [codes.graveAccent]: commonmark.codeFenced,
+        [codes.tilde]: commonmark.codeFenced
+    },
+    flowInitial: {
+        [codes.horizontalTab]: commonmark.codeIndented,
+        [codes.virtualSpace]: commonmark.codeIndented,
+        [codes.space]: commonmark.codeIndented
+    },
+    insideSpan: {
+        null: [
+            { resolveAll: commonmark.attention.resolveAll },
+            { resolveAll: string.resolveAll }
+        ]
+    },
+    string: {
+        [codes.ampersand]: commonmark.characterReference,
+        [codes.backslash]: commonmark.characterEscape
+    },
+    text: {
+        [codes.carriageReturn]: commonmark.lineEnding,
+        [codes.lineFeed]: commonmark.lineEnding,
+        [codes.carriageReturnLineFeed]: commonmark.lineEnding,
+        [codes.exclamationMark]: commonmark.labelStartImage,
+        [codes.ampersand]: commonmark.characterReference,
+        [codes.asterisk]: commonmark.attention,
+        [codes.lessThan]: [commonmark.autolink, commonmark.htmlText],
+        [codes.leftSquareBracket]: commonmark.labelStartLink,
+        [codes.backslash]: [commonmark.hardBreakEscape, commonmark.characterEscape],
+        [codes.rightSquareBracket]: commonmark.labelEnd,
+        [codes.underscore]: commonmark.attention,
+        [codes.graveAccent]: commonmark.codeText
+    }
+};
+
+function resolveRegionExits(events) {
+    let index = -1;
+    let region;
+    while (++index < events.length) {
+        ok(events[index], 'expected `events[index]`');
+        const [event, token, self] = events[index];
+        if (token._region) {
+            if (event === ev.enter) {
+                region = token;
+            }
+            else {
+                region = undefined;
+                const previousIndex = index - 1;
+                ok(events[previousIndex], 'expected `events[previousIndex]`');
+                const [previousEvent, previousToken] = events[previousIndex];
+                if (previousEvent === ev.exit &&
+                    (previousToken.type === tt.hardBreakEscape ||
+                        previousToken.type === tt.hardBreakTrailing ||
+                        previousToken.type === tt.lineSuffix ||
+                        previousToken._trailing)) {
+                    token.end = structuredClone(previousToken.start);
+                    splice(events, index, 1);
+                    splice(events, previousIndex - 1, 0, [[event, token, self]]);
+                    continue;
+                }
+            }
+        }
+        if (region && event === ev.enter && token.type === tt.lineEnding) {
+            let position = index + 1;
+            while (++position < events.length) {
+                ok(events[position], 'expected `events[position]`');
+                ok(token !== events[position][1], 'did not expect `token` match');
+                const event = events[position];
+                if (region === event[1]) {
+                    ok(event[0] === ev.exit, 'expected region `exit` event');
+                    event[1].end = structuredClone(token.start);
+                    splice(events, position, 1);
+                    splice(events, index, 0, [event]);
+                    index--;
+                    break;
+                }
+                if (event[0] === ev.enter) {
+                    if (event[1].type === tt.linePrefix) {
+                        const k = 2;
+                        if (events[position + k]?.[1].type !== tt.lineEndingBlank) {
+                            index++;
+                            break;
+                        }
+                        else {
+                            position += k;
+                        }
+                    }
+                    else if (event[1].type !== token.type &&
+                        event[1].type !== tt.lineEndingBlank) {
+                        index++;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    return events;
+}
+
+function postprocess$1(events) {
+    while (!subtokenize(events))
+        ;
+    return resolveRegionExits(events);
+}
+
+const comment = {
+    resolve: resolveComment,
+    tokenize: tokenizeComment
+};
+function resolveComment(events) {
+    return postprocess$1(events);
+}
+function tokenizeComment(effects) {
+    const self = this;
+    const stack = [];
+    let continued = 0;
+    let markdown;
+    let md;
+    let then;
+    return start;
+    function start(code) {
+        ok(bos(self.previous) || eol(self.previous) || eos(code), 'expected beginning of stream, beginning of line, or end of stream');
+        restate();
+        if (!stack[0]) {
+            return effects.check(blankLine, beforeBlankLine, checkNewRegions)(code);
+        }
+        return effects.check(blankLine, continueAtBlankLine, continueAtNoBlankLine)(code);
+    }
+    function beforeBlankLine(code) {
+        self.parser.atBlankLine = true;
+        return beforeMarkdown(code);
+    }
+    function continueAtBlankLine(code) {
+        ok(stack.length, 'expected region on `stack`');
+        self.parser.atBlankLine = true;
+        return tryContinuation(code);
+    }
+    function continueAtNoBlankLine(code) {
+        ok(stack.length, 'expected region on `stack`');
+        self.parser.atBlankLine = false;
+        return checkNewRegions(code);
+    }
+    function checkNewRegions(code) {
+        ok(!self.parser.atBlankLine, 'expected no blank line');
+        self.interrupt = Boolean(markdown?.currentConstruct ?? markdown?.interrupt);
+        if (stack.length === 1) {
+            if (!markdown) {
+                return effects.check(region, aNewRegion, tryContinuation)(code);
+            }
+            if (markdown.concrete)
+                return tryContinuation(code);
+            return effects.check(region, aNewRegion, tryContinuation)(code);
+        }
+        ok(!stack.length, 'expected no region on `stack`');
+        return tryRegion(code);
+    }
+    function aNewRegion(code) {
+        ok(stack.length === 1, 'expected region on `stack`');
+        flush();
+        return tryRegion(code);
+    }
+    function tryRegion(code) {
+        restate();
+        continued = self.events.length;
+        return effects.attempt(region, addRegion, beforeMarkdown)(code);
+    }
+    function addRegion(code) {
+        ok(self.currentConstruct, 'expected a current construct');
+        ok(self.currentConstruct.continuation, 'expected continuable construct');
+        ok(self.containerState, 'expected `containerState` when adding region');
+        forward();
+        ok(stack.length === 0, 'expected empty region stack');
+        stack[0] = [self.currentConstruct, self.containerState];
+        self.parser.freshRegion = true;
+        self.parser.skipSummary = true;
+        if (self.containerState._closeFlow)
+            flush();
+        if (bos(self.previous) || eol(self.previous))
+            return start(code);
+        if (eol(code)) {
+            effects.enter(tt.lineEnding);
+            effects.consume(code);
+            effects.exit(tt.lineEnding);
+            return start;
+        }
+        ok(eos(code), 'expected end of stream');
+        return start(code);
+    }
+    function tryContinuation(code) {
+        ok(self.containerState, 'expected `containerState` when continuing');
+        ok(stack[0], 'expected region on `stack` when continuing');
+        const [construct, containerState] = stack[0];
+        self.containerState = containerState;
+        continued = self.events.length;
+        then = self.now();
+        self.interrupt = Boolean(markdown?.currentConstruct ?? markdown?.interrupt);
+        return effects.attempt(construct.continuation, afterContinuation, noContinuation)(code);
+    }
+    function afterContinuation(code) {
+        ok(self.containerState, 'expected `containerState` after continuing');
+        ok(stack.length === 1, 'expected region on `stack`');
+        ok(then, 'expected `then` after continuing');
+        forward();
+        if (self.containerState._closeFlow)
+            return noContinuation(code);
+        self.parser.freshRegion = false;
+        const now = self.now();
+        if (then.line === now.line &&
+            then.column === now.column &&
+            then.offset === now.offset &&
+            then._bufferIndex === now._bufferIndex &&
+            then._index === now._index) {
+            return beforeMarkdown(code);
+        }
+        if (!eol(self.previous))
+            return beforeMarkdown(code);
+        return start(code);
+    }
+    function noContinuation(code) {
+        flush();
+        if (self.parser.atBlankLine)
+            return beforeMarkdown(code);
+        return start(code);
+    }
+    function beforeMarkdown(code) {
+        if (eos(code))
+            return void end(code);
+        return markdownStart(code);
+    }
+    function markdownStart(code) {
+        ok(!eos(code), 'did not expect end of stream');
+        markdown ??= self.parser.document(self.now());
+        effects.enter(tt.chunkMarkdown, {
+            _tokenizer: markdown,
+            contentType: constants.contentTypeDocument,
+            previous: md
+        });
+        return markdownContinue(code);
+    }
+    function markdownContinue(code) {
+        ok(markdown, 'expected `markdown` when continuing markdown');
+        ok(self.containerState, 'expected `containerState` inside markdown');
+        if (eos(code)) {
+            write(effects.exit(tt.chunkMarkdown), true);
+            return void end(code);
+        }
+        if (eol(code)) {
+            effects.consume(code);
+            write(effects.exit(tt.chunkMarkdown));
+            return start;
+        }
+        effects.consume(code);
+        return markdownContinue;
+    }
+    function closeMarkdown() {
+        if (markdown && !eos(markdown.code))
+            markdown.write([codes.eos]);
+        markdown = undefined;
+        md = undefined;
+        return void markdown;
+    }
+    function end(code) {
+        ok(eos(code), 'expected end of stream');
+        flush();
+        self.parser.atBlankLine = undefined;
+        self.parser.freshRegion = undefined;
+        self.parser.previousBlankLine = undefined;
+        return void effects.consume(code);
+    }
+    function flush() {
+        ok(stack.length <= 1, 'expected no more than 1 region');
+        closeMarkdown();
+        while (stack.length) {
+            const [construct, containerState] = stack.pop();
+            self.containerState = containerState;
+            construct.exit.call(self, effects);
+        }
+        self.currentConstruct = undefined;
+        self.concrete = undefined;
+        return void restate();
+    }
+    function forward() {
+        while (continued < self.events.length) {
+            ok(self.events[continued], 'expected `self.events[continued]`');
+            const [event, token] = self.events[continued];
+            if (event === ev.exit &&
+                token.type === tt.chunkMarkdown &&
+                token.contentType === constants.contentTypeDocument) {
+                write(token);
+            }
+            continued++;
+        }
+        return void self.events;
+    }
+    function restate() {
+        const { comment } = self.containerState ?? {};
+        self.containerState = {};
+        self.containerState.comment = comment;
+        return void self.containerState;
+    }
+    function write(token, end) {
+        ok(self.containerState, 'expected `containerState` during write');
+        ok(token !== md, 'did not expect `token` to match previous token');
+        markdown ??= self.parser.document(token.start);
+        token._tokenizer ??= markdown;
+        const stream = self.sliceStream(token);
+        if (eol(stream.at(-1))) {
+            self.parser.previousBlankLine = self.parser.atBlankLine;
+            self.parser.atBlankLine = null;
+        }
+        if (end)
+            stream.push(codes.eos);
+        token.previous = md;
+        if (md)
+            md.next = token;
+        md = token;
+        if (token.previous)
+            markdown.defineSkip(token.start);
+        markdown.write(stream);
+        self.concrete = markdown.concrete;
+        return void markdown;
+    }
+}
+
+const content = { tokenize: tokenizeContent };
+function tokenizeContent(effects) {
+    const contentStart = effects.attempt(this.parser.constructs.contentInitial, afterContentStart, paragraphInitial);
+    let previous;
+    return contentStart;
+    function afterContentStart(code) {
+        if (eos(code))
+            return void effects.consume(code);
+        ok(eol(code), 'expected eol');
+        effects.enter(tt.lineEnding);
+        effects.consume(code);
+        effects.exit(tt.lineEnding);
+        return factorySpace(effects, contentStart, tt.linePrefix);
+    }
+    function paragraphInitial(code) {
+        ok(!eos(code) && !eol(code), 'expected anything other than a line ending or eof');
+        effects.enter(tt.paragraph);
+        return lineStart(code);
+    }
+    function lineStart(code) {
+        const token = effects.enter(tt.chunkText, {
+            contentType: ct.text,
+            previous
+        });
+        if (previous)
+            previous.next = token;
+        previous = token;
+        return data(code);
+    }
+    function data(code) {
+        if (eos(code)) {
+            effects.exit(tt.chunkText);
+            effects.exit(tt.paragraph);
+            return void effects.consume(code);
+        }
+        if (eol(code)) {
+            effects.consume(code);
+            effects.exit(tt.chunkText);
+            return lineStart;
+        }
+        effects.consume(code);
+        return data;
+    }
+}
+
+function postprocess(events) {
+  while (!subtokenize$1(events)) {
+  }
+  return events;
+}
+
+function resolveMarkdown(events) {
+    return postprocess(events);
+}
+
+const document = {
+    resolve: resolveMarkdown,
+    tokenize: tokenizeDocument
+};
+const container = { tokenize: tokenizeContainer };
+function tokenizeDocument(effects) {
+    const self = this;
+    const stack = [];
+    let childFlow;
+    let childToken;
+    let continued = 0;
+    let lineStartOffset;
+    return start;
+    function start(code) {
+        if (continued < stack.length) {
+            const [construct, containerState] = stack[continued];
+            ok(construct.continuation, 'expected `continuation` on container construct');
+            self.containerState = containerState;
+            return effects.attempt(construct.continuation, documentContinue, checkNewContainers)(code);
+        }
+        return checkNewContainers(code);
+    }
+    function documentContinue(code) {
+        ok(self.containerState, 'expected `containerState` after continuation');
+        continued++;
+        if (self.containerState._closeFlow) {
+            self.containerState._closeFlow = undefined;
+            closeFlow();
+            const indexBeforeExits = self.events.length;
+            let flowExitIndex = indexBeforeExits;
+            let point = undefined;
+            while (flowExitIndex--) {
+                ok(self.events[flowExitIndex], 'expected `events[flowExitIndex]`');
+                const [event, token] = self.events[flowExitIndex];
+                if (event === ev.exit && token.type === tt.chunkFlow) {
+                    point = token.end;
+                    break;
+                }
+            }
+            ok(point, 'could not find previous flow chunk');
+            exitContainers(continued);
+            let index = indexBeforeExits;
+            while (index < self.events.length) {
+                self.events[index][1].end = { ...point };
+                index++;
+            }
+            splice(self.events, flowExitIndex + 1, 0, self.events.slice(indexBeforeExits));
+            self.events.length = index;
+            return checkNewContainers(code);
+        }
+        return start(code);
+    }
+    function checkNewContainers(code) {
+        if (continued === stack.length) {
+            if (!childFlow)
+                return documentContinued(code);
+            if (childFlow.currentConstruct?.concrete)
+                return flowStart(code);
+            self.interrupt = Boolean(childFlow.currentConstruct && !childFlow._gfmTableDynamicInterruptHack);
+        }
+        self.containerState = {};
+        return effects.check(container, thereIsANewContainer, thereIsNoNewContainer)(code);
+    }
+    function thereIsANewContainer(code) {
+        closeFlow();
+        exitContainers(continued);
+        return documentContinued(code);
+    }
+    function thereIsNoNewContainer(code) {
+        self.parser.lazy[self.now().line] = continued !== stack.length;
+        lineStartOffset = self.now().offset;
+        return flowStart(code);
+    }
+    function documentContinued(code) {
+        self.containerState = {};
+        return effects.attempt(container, containerContinue, flowStart)(code);
+    }
+    function containerContinue(code) {
+        ok(self.containerState, 'expected `containerState`');
+        ok(self.currentConstruct, 'expected `currentConstruct`');
+        ok(self.currentConstruct.continuation, 'expected container construct');
+        continued++;
+        stack.push([self.currentConstruct, self.containerState]);
+        return documentContinued(code);
+    }
+    function flowStart(code) {
+        if (eos(code)) {
+            closeFlow();
+            exitContainers(0);
+            return void effects.consume(code);
+        }
+        childFlow ??= self.parser.flow(self.now());
+        effects.enter(tt.chunkFlow, {
+            _tokenizer: childFlow,
+            contentType: constants.contentTypeFlow,
+            previous: childToken
+        });
+        return flowContinue(code);
+    }
+    function flowContinue(code) {
+        if (eos(code)) {
+            writeToChild(effects.exit(tt.chunkFlow), true);
+            exitContainers(0);
+            return void effects.consume(code);
+        }
+        if (eol(code)) {
+            effects.consume(code);
+            writeToChild(effects.exit(tt.chunkFlow));
+            continued = 0;
+            self.interrupt = undefined;
+            return start;
+        }
+        effects.consume(code);
+        return flowContinue;
+    }
+    function exitContainers(size) {
+        let index = stack.length;
+        while (index-- > size) {
+            ok(stack[index], 'expected `stack[index]`');
+            const [construct, containerState] = stack[index];
+            self.containerState = containerState;
+            ok(construct.exit, 'expected `exit` on container construct');
+            construct.exit.call(self, effects);
+        }
+        stack.length = size;
+        if (!stack.length)
+            self.currentConstruct = undefined;
+        return void size;
+    }
+    function writeToChild(token, end) {
+        ok(childFlow, 'expected `childFlow` to be defined when continuing');
+        const stream = self.sliceStream(token);
+        if (end)
+            stream.push(codes.eos);
+        token.previous = childToken;
+        if (childToken)
+            childToken.next = token;
+        childToken = token;
+        childFlow.defineSkip(token.start);
+        childFlow.write(stream);
+        self.concrete = childFlow.currentConstruct?.concrete;
+        if (self.parser.lazy[token.start.line]) {
+            let index = childFlow.events.length;
+            while (index--) {
+                const [, eventToken] = childFlow.events[index];
+                if (
+                eventToken.start.offset < lineStartOffset && (
+                !eventToken.end ||
+                    eventToken.end.offset > lineStartOffset
+                )) {
+                    return void eventToken;
+                }
+            }
+            const indexBeforeExits = self.events.length;
+            let flowExitIndex = indexBeforeExits;
+            let point = undefined;
+            let seen = undefined;
+            while (flowExitIndex--) {
+                ok(self.events[flowExitIndex], 'expected `events[flowExitIndex]`');
+                const [event, token] = self.events[flowExitIndex];
+                if (event === ev.exit && token.type === tt.chunkFlow) {
+                    if (seen) {
+                        point = token.end;
+                        break;
+                    }
+                    seen = true;
+                }
+            }
+            ok(point, 'could not find previous flow chunk');
+            exitContainers(continued);
+            index = indexBeforeExits;
+            while (index < self.events.length) {
+                self.events[index][1].end = { ...point };
+                index++;
+            }
+            splice(self.events, flowExitIndex + 1, 0, self.events.slice(indexBeforeExits));
+            self.events.length = index;
+        }
+        return void token;
+    }
+    function closeFlow() {
+        ok(self.containerState, 'expected `containerState` when closing flow');
+        if (childFlow) {
+            childFlow.write([codes.eos]);
+            childFlow = undefined;
+            childToken = undefined;
+            self.containerState._closeFlow = undefined;
+        }
+        return void childFlow;
+    }
+}
+function tokenizeContainer(effects, ok$1, nok) {
+    ok(this.parser.constructs.disable.null, 'expected `disable.null`');
+    return factorySpace(effects, effects.attempt(this.parser.constructs.document, ok$1, nok), tt.linePrefix, this.parser.constructs.disable.null.includes('codeIndented')
+        ? undefined
+        : constants.tabSize);
+}
+
+const flow = { tokenize: tokenizeFlow };
+function tokenizeFlow(effects) {
+    const self = this;
+    return initial;
+    function initial(code) {
+        return effects.attempt(blankLine,
+        atBlankEnding, noBlankLine)(code);
+    }
+    function noBlankLine(code) {
+        return effects.attempt(self.parser.constructs.flowInitial,
+        afterFlow, noFlowInitial)(code);
+    }
+    function noFlowInitial(code) {
+        return factorySpace(effects, effects.attempt(self.parser.constructs.flow, afterFlow, noFlow), tt.linePrefix)(code);
+    }
+    function noFlow(code) {
+        return effects.attempt(commonmark.content, afterFlow)(code);
+    }
+    function atBlankEnding(code) {
+        if (eos(code))
+            return void effects.consume(code);
+        ok(eol(code), 'expected eol');
+        effects.enter(tt.lineEndingBlank);
+        effects.consume(code);
+        effects.exit(tt.lineEndingBlank);
+        self.currentConstruct = undefined;
+        return initial;
+    }
+    function afterFlow(code) {
+        if (eos(code))
+            return void effects.consume(code);
+        ok(eol(code), 'expected eol');
+        effects.enter(tt.lineEnding);
+        effects.consume(code);
+        effects.exit(tt.lineEnding);
+        self.currentConstruct = undefined;
+        return initial;
+    }
+}
+
+const source = { tokenize: tokenizeSource };
+const sourceComment = { tokenize: tokenizeSourceComment };
+function tokenizeSource(effects) {
+    const self = this;
+    const stack = [];
+    let comment;
+    let content;
+    let continued = 0;
+    return start;
+    function start(code) {
+        ok(bos(self.previous) || eol(self.previous), 'expected beginning of stream or line');
+        self.containerState ??= {};
+        if (!stack[0])
+            return effects.attempt(blankLine, restart, tryComment)(code);
+        return tryContinuation(code);
+    }
+    function tryContinuation(code) {
+        ok(self.containerState, 'expected `containerState` when continuing');
+        ok(stack[0], 'expected comment on `stack` when continuing');
+        const [construct, containerState] = stack[0];
+        ok(self.containerState === containerState, 'expected `containerState` to match `stack[0][1]`');
+        continued = self.events.length;
+        self.interrupt = Boolean(comment?.currentConstruct ?? comment?.interrupt);
+        return effects.attempt(construct.continuation, afterContinuation, noContinuation)(code);
+    }
+    function afterContinuation(code) {
+        ok(stack.length === 1, 'expected comment on `stack`');
+        ok(self.containerState, 'expected `containerState` after continuing');
+        forward();
+        if (self.containerState._closeFlow)
+            return noContinuation(code);
+        self.parser.freshComment = false;
+        if (!eol(self.previous))
+            return beforeChunk(code);
+        return start(code);
+    }
+    function noContinuation(code) {
+        flush();
+        return tryComment(code);
+    }
+    function tryComment(code) {
+        self.containerState = {};
+        continued = self.events.length;
+        if (whitespace(code))
+            return factorySpace(effects, tryComment)(code);
+        return effects.attempt(sourceComment, takeComment, restart)(code);
+    }
+    function takeComment(code) {
+        ok(self.containerState, 'expected `containerState`');
+        ok(self.currentConstruct, 'expected `currentConstruct`');
+        ok(self.currentConstruct.continuation, 'expected continuable construct');
+        ok(self.events[continued], 'expected `self.events[continued]`');
+        ok(self.events[continued][0] === ev.enter, 'expected `enter` event');
+        self.containerState.comment ??= self.events[continued][1]._kind;
+        forward();
+        ok(stack.length === 0, 'expected empty comment stack');
+        stack[0] = [self.currentConstruct, self.containerState];
+        self.parser.freshComment = true;
+        if (self.containerState._closeFlow)
+            return noContinuation(code);
+        if (eol(self.previous))
+            return start(code);
+        return beforeChunk(code);
+    }
+    function restart(code) {
+        ok(!comment, 'did not expect comment content parser');
+        ok(stack.length === 0, 'expected empty comment stack');
+        if (eos(code))
+            return beforeChunk(code);
+        effects.consume(code);
+        if (eol(self.previous))
+            return start;
+        return tryComment;
+    }
+    function beforeChunk(code) {
+        if (eos(code))
+            return void end(code);
+        return chunkStart(code);
+    }
+    function chunkStart(code) {
+        ok(!eos(code), 'did not expect end of stream');
+        ok(self.containerState, 'expected `containerState` inside comment');
+        ok(self.containerState.comment, 'expected comment kind');
+        ok(stack.length === 1, 'expected comment on `stack`');
+        comment ??= self.parser.comment(self.now());
+        comment.containerState ??= {};
+        comment.containerState.comment = self.containerState.comment;
+        effects.enter(tt.chunkComment, {
+            _tokenizer: comment,
+            contentType: constants.contentTypeComment,
+            previous: content
+        });
+        return chunkContinue(code);
+    }
+    function chunkContinue(code) {
+        ok(comment, 'expected comment content parser');
+        if (eos(code)) {
+            write(effects.exit(tt.chunkComment), true);
+            return void end(code);
+        }
+        if (eol(code)) {
+            effects.consume(code);
+            write(effects.exit(tt.chunkComment));
+            self.interrupt = undefined;
+            return start;
+        }
+        effects.consume(code);
+        return chunkContinue;
+    }
+    function end(code) {
+        ok(eos(code), 'expected end of stream');
+        flush();
+        self.parser.freshComment = undefined;
+        effects.enter(tt.eoc);
+        effects.consume(code);
+        effects.exit(tt.eoc);
+        return void code;
+    }
+    function flush() {
+        ok(stack.length <= 1, 'expected no more than 1 comment');
+        if (comment && !eos(comment.code))
+            comment.write([codes.eos]);
+        comment = undefined;
+        content = undefined;
+        while (stack.length) {
+            const [construct, containerState] = stack.pop();
+            self.containerState = containerState;
+            construct.exit.call(self, effects);
+        }
+        self.currentConstruct = undefined;
+        self.containerState = {};
+        self.parser.skipSummary = undefined;
+        return void stack;
+    }
+    function forward() {
+        while (continued < self.events.length) {
+            ok(self.events[continued], 'expected `self.events[continued]`');
+            const [event, token] = self.events[continued];
+            if (event === ev.exit &&
+                token.type === tt.chunkComment &&
+                token.contentType === constants.contentTypeComment) {
+                write(token);
+            }
+            continued++;
+        }
+        return void self.events;
+    }
+    function write(token, end) {
+        ok(self.containerState, 'expected `containerState` during write');
+        ok(token !== content, 'did not expect `token` to match previous token');
+        comment ??= self.parser.comment(token.start);
+        token._tokenizer ??= comment;
+        const stream = self.sliceStream(token);
+        if (end)
+            stream.push(codes.eos);
+        token.previous = content;
+        if (content)
+            content.next = token;
+        content = token;
+        if (token.previous)
+            comment.defineSkip(token.start);
+        comment.write(stream);
+        self.concrete = comment.concrete;
+        return void token;
+    }
+}
+function tokenizeSourceComment(effects, ok, nok) {
+    return effects.attempt(this.parser.constructs.source, ok, nok);
+}
+
+const type = { tokenize: tokenizeType };
+function tokenizeType(effects) {
+    const self = this;
+    return attempt;
+    function attempt(code) {
+        return effects.attempt(self.parser.constructs.type, attempt, eat)(code);
+    }
+    function eat(code) {
+        return effects.consume(code), attempt;
+    }
+}
+
+function parse(options) {
+    const context = createTokenizer({
+        debug: 'docmark',
+        eol,
+        extensions: extensions,
+        finalizeContext,
+        initialize: initialize
+    });
+    return context.parser;
+    function extensions() {
+        return combineExtensions(docmark, markdown, options?.extensions);
+    }
+    function finalizeContext(self) {
+        if (typeof self.parser.defined === 'undefined')
+            self.parser.defined = [];
+        if (typeof self.parser.lazy === 'undefined')
+            self.parser.lazy = {};
+        switch (self.contentType) {
+            case constants.contentTypeDocument:
+            case constants.contentTypeFlow:
+            case constants.contentTypeContent:
+            case constants.contentTypeText:
+            case constants.contentTypeString:
+                self.code = codes.eos;
+                self.previous = codes.eos;
+                self.noEmptyTokens = true;
+                self.noPrevious = true;
+                break;
+            case constants.contentTypeSource:
+            case constants.contentTypeComment:
+            case constants.contentTypeType:
+                self.code = codes.bos;
+                self.previous = codes.bos;
+                break;
+        }
+        if (self.contentType) {
+            options?.finalizeContext?.(self);
+        }
+        return void self;
+    }
+    function initialize() {
+        return {
+            [constants.contentTypeSource]: source,
+            [constants.contentTypeComment]: comment,
+            [constants.contentTypeType]: type,
+            [constants.contentTypeDocument]: document,
+            [constants.contentTypeFlow]: flow,
+            [constants.contentTypeContent]: content,
+            [constants.contentTypeText]: text,
+            [constants.contentTypeString]: string,
+            ...options?.initializers
+        };
+    }
+}
+
+function preprocess(options) {
+    const tabSize = options?.tabSize ?? constants.tabSize;
+    return preprocessor;
+    function preprocessor(value, encoding, end) {
+        const chunks = [];
+        if (typeof value === 'number') {
+            if (value !== codes.bos && value !== codes.empty)
+                chunks.push(value);
+            if (end)
+                chunks.push(codes.eos);
+            return chunks;
+        }
+        if (value !== codes.eos && value !== undefined) {
+            let decoded = decode(value, encoding);
+            if (typeof decoded === 'number') {
+                if (end)
+                    chunks.push(codes.eos);
+                return chunks;
+            }
+            value = decoded;
+            let column = 1;
+            let index = 0;
+            let sliceIndex = 0;
+            while (index < value.length) {
+                const code = value.codePointAt(index);
+                let k = code > 0xffff ? 2 : 1;
+                if (code !== codes.nul && !eol(code) && !htab(code)) {
+                    index += k;
+                    continue;
+                }
+                if (sliceIndex < index) {
+                    const chunk = value.slice(sliceIndex, index);
+                    chunks.push(chunk);
+                    column += [...chunk].length;
+                }
+                switch (code) {
+                    case codes.ht:
+                        const n = Math.ceil(column / tabSize) * tabSize;
+                        chunks.push(codes.horizontalTab);
+                        while (column++ < n)
+                            chunks.push(codes.virtualSpace);
+                        break;
+                    case codes.lf:
+                        chunks.push(codes.lineFeed);
+                        column = 1;
+                        break;
+                    case codes.nul:
+                        chunks.push(codes.nul);
+                        column = 1;
+                        break;
+                    default:
+                        const nextIndex = index + 1;
+                        const next = value.codePointAt(nextIndex);
+                        if (next === codes.lf) {
+                            chunks.push(codes.carriageReturnLineFeed);
+                            index = nextIndex;
+                        }
+                        else {
+                            chunks.push(codes.carriageReturn);
+                        }
+                        column = 1;
+                        break;
+                }
+                index += k;
+                sliceIndex = index;
+            }
+            if (sliceIndex < value.length)
+                chunks.push(value.slice(sliceIndex));
+        }
+        return end && chunks.push(codes.eos), chunks;
+    }
+}
+
+export { parse, postprocess$1 as postprocess, preprocess };

package/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "@flex-development/docmark",
+  "description": "commonmark compliant documentation parser with positional info and concrete tokens",
+  "version": "1.0.0-dev.3",
+  "keywords": [
+    "comment",
+    "commonmark",
+    "compiler",
+    "docblock",
+    "docdown",
+    "documentation",
+    "gfm",
+    "html",
+    "jsdoc",
+    "lexer",
+    "markdown",
+    "md",
+    "mdast",
+    "parse",
+    "parser",
+    "token",
+    "tokenize",
+    "tokenizer",
+    "tsdoc",
+    "typedoc",
+    "typescript",
+    "unified"
+  ],
+  "license": "BSD-3-Clause",
+  "homepage": "https://github.com/flex-development/docmark",
+  "repository": {
+    "directory": "packages/docmark",
+    "type": "git",
+    "url": "git+https://github.com/flex-development/docmark.git"
+  },
+  "bugs": "https://github.com/flex-development/docmark/issues",
+  "author": {
+    "name": "Lexus Drumgold",
+    "url": "https://github.com/unicornware"
+  },
+  "publishConfig": {
+    "access": "public",
+    "diff-dst-prefix": "docmark",
+    "diff-src-prefix": "docmark",
+    "directory": "./",
+    "executableFiles": [],
+    "node-options": null,
+    "pack-destination": ".",
+    "parseable": true,
+    "prefer-dedupe": true,
+    "provenance": true,
+    "tag-version-prefix": "docmark"
+  },
+  "type": "module",
+  "files": [
+    "LICENSE.md",
+    "README.md",
+    "dist"
+  ],
+  "exports": {
+    ".": {
+      "docmark": "./src/index.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "imports": {
+    "#fixtures/*": "./__fixtures__/*.mts",
+    "#lib/postprocess": {
+      "docmark": "./src/postprocess.mts",
+      "default": "./dist/postprocess.mjs"
+    },
+    "#tests/*": "./__tests__/*.mts"
+  },
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "scripts": {
+    "attw": "tsc --build tsconfig.build.json --clean --dry && tsc --build tsconfig.build.json --clean && tsc --build tsconfig.build.json --verbose"
+  },
+  "dependencies": {
+    "@flex-development/docmark-factory-space": "1.0.0-dev.3",
+    "@flex-development/docmark-grammar": "1.0.0-dev.3",
+    "@flex-development/docmark-util-combine-extensions": "1.0.0-dev.3",
+    "@flex-development/docmark-util-subtokenize": "1.0.0-dev.3",
+    "@flex-development/docmark-util-symbol": "1.0.0-dev.3",
+    "@flex-development/docmark-util-types": "1.0.0-dev.3",
+    "@flex-development/mark": "1.0.0-alpha.4",
+    "@flex-development/mark-parser": "1.0.0-alpha.19",
+    "@flex-development/mark-util-character": "1.1.0",
+    "@flex-development/mark-util-chunked": "1.0.0",
+    "devlop": "1.1.0",
+    "micromark-core-commonmark": "2.0.3",
+    "micromark-util-subtokenize": "2.1.0",
+    "micromark-util-types": "2.0.2"
+  },
+  "sideEffects": false,
+  "build": "575af951968e8cbab57769b5789f5542b71d62be"
+}