@flex-development/docmark-util-subtokenize 1.0.0-alpha.1

package/LICENSE.md

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Flex Development, LLC All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/README.md

@@ -0,0 +1,86 @@
+# docmark-util-subtokenize
+
+[![npm](https://img.shields.io/npm/v/@flex-development/docmark-util-subtokenize.svg)](https://npmjs.com/package/@flex-development/docmark-util-subtokenize)
+[![npm downloads](https://img.shields.io/npm/dm/@flex-development/docmark-util-subtokenize.svg)](https://www.npmcharts.com/compare/@flex-development/docmark-util-subtokenize?interval=30)
+[![install size](https://packagephobia.now.sh/badge?p=@flex-development/docmark-util-subtokenize)](https://packagephobia.now.sh/result?p=@flex-development/docmark-util-subtokenize)
+[![module type: esm](https://img.shields.io/badge/module%20type-esm-brightgreen)](https://github.com/voxpelli/badges-cjs-esm)
+
+[docmark][] utility to tokenize subtokens.
+
+## Contents
+
+- [What is this?](#what-is-this)
+- [When should I use this?](#when-should-i-use-this)
+- [Install](#install)
+- [Use](#use)
+- [API](#api)
+- [Types](#types)
+- [Contribute](#contribute)
+
+## What is this?
+
+**TODO**: what is this?
+
+## When should I use this?
+
+This package is useful when extending docmark.
+
+## Install
+
+This package is [ESM only][esm].
+
+In Node.js with [yarn][]:
+
+```sh
+yarn add @flex-development/docmark-util-subtokenize
+```
+
+<blockquote>
+  <small>
+    See <a href='https://yarnpkg.com/protocol/git'>Git - Protocols | Yarn</a>
+    &nbsp;for details regarding installing from Git.
+  </small>
+</blockquote>
+
+In Deno with [`esm.sh`][esmsh]:
+
+```ts
+import { subcontent, subtokenize } from 'https://esm.sh/@flex-development/docmark-util-subtokenize'
+```
+
+In browsers with [`esm.sh`][esmsh]:
+
+```html
+<script type="module">
+  import { subcontent, subtokenize } from 'https://esm.sh/@flex-development/docmark-util-subtokenize'
+</script>
+```
+
+## Use
+
+**TODO**: use
+
+## API
+
+**TODO**: api
+
+## Types
+
+This package is fully typed with [TypeScript][].
+
+## Contribute
+
+See [`CONTRIBUTING.md`](../../CONTRIBUTING.md).
+
+This project has a [code of conduct](../../CODE_OF_CONDUCT.md).
+By interacting with this repository, organization, or community you agree to abide by its terms.
+
+[docmark]: ../../README.md
+
+[esm]: https://gist.github.com/sindresorhus/a39789f98801d908bbc7ff3ecc99d99c
+
+[esmsh]: https://esm.sh
+
+[typescript]: https://www.typescriptlang.org
+
+[yarn]: https://yarnpkg.com

package/dist/index.d.mts

@@ -0,0 +1,64 @@
+import type { Event } from '@flex-development/docmark-util-types';
+
+/**
+ * @file subcontent
+ * @module docmark-util-subtokenize/subcontent
+ */
+
+/**
+ * Tokenize embedded content for a single token.
+ *
+ * This algorithm has three phases:
+ *
+ * 1. Feed linked chunk tokens to a child tokenizer
+ * 2. Determine which child events belong to each linked token
+ * 3. Replace `chunk*` events with their corresponding child events
+ *
+ * @this {void}
+ *
+ * @param {Event[]} events
+ *  The parent event stream
+ * @param {number} eventIndex
+ *  The index of the first chunk `enter` event
+ * @return {undefined}
+ */
+declare function subcontent(this: void, events: Event[], eventIndex: number): undefined;
+
+/**
+ * @file subtokenize
+ * @module docmark-util-subtokenize/subtokenize
+ */
+
+/**
+ * Tokenize embedded content.
+ *
+ * Some tokens declare a {@linkcode ContentType}.
+ * These tokens do not contain fully parsed content themselves.
+ * Tokens with a `chunk*` (i.e. `chunkMarkdown`, `chunkDocument`, `chunkFlow`)
+ * type and a `contentType` act as containers for another tokenizer.
+ *
+ * For example:
+ *
+ * ```text
+ * comment
+ * └─ chunkComment
+ * ```
+ *
+ * A `chunkComment` token may contain markdown, block tags, inline tags, and
+ * other syntax. This function replaces those chunk tokens with the events
+ * produced by their child tokenizer.
+ *
+ * Linked chunks are handled as one logical stream, but child events are spliced
+ * back into the same positions as the original chunk tokens. This preserves
+ * surrounding events such as ones for `commentLinePrefix`.
+ *
+ * @this {void}
+ *
+ * @param {Event[]} events
+ *  The current list of events
+ * @return {boolean}
+ *  Whether subtokens (embedded content) were found
+ */
+declare function subtokenize(this: void, events: Event[]): boolean;
+
+export { subcontent, subtokenize };

package/dist/index.mjs

@@ -0,0 +1,120 @@
+import { codes, ev } from '@flex-development/docmark-util-symbol';
+import { splice } from '@flex-development/mark-util-chunked';
+import { ok } from 'devlop';
+
+function eolset(code, start) {
+    return start + (code === codes.carriageReturnLineFeed ? 2 : 1);
+}
+
+function subcontent(events, eventIndex) {
+    ok(events[eventIndex], 'expected `events[eventIndex]`');
+    const [event, token, context] = events[eventIndex];
+    ok(event === ev.enter, 'expected enter event');
+    ok(token.contentType, 'expected `token.contentType`');
+    ok(/^chunk[A-Z][a-z]+$/.test(token.type), 'expected chunk token');
+    const child = token._tokenizer && 'write' in token._tokenizer
+        ? token._tokenizer
+        : context.parser[token.contentType](token.start);
+    const positions = [];
+    let current = token;
+    let position = eventIndex - 1;
+    let previous;
+    while (current) {
+        while (++position < events.length) {
+            ok(events[position], 'expected `events[j]`');
+            const [event, eventToken] = events[position];
+            if (event === ev.enter && eventToken === current) {
+                positions.push(position);
+                break;
+            }
+        }
+        ok(position < events.length, 'expected `enter` event for linked token');
+        ok(!previous || current.previous === previous, 'expected previous to match');
+        ok(!previous || previous.next === current, 'expected next to match');
+        if (!current._tokenizer) {
+            const chunks = context.sliceStream(current);
+            if (current.previous)
+                child.defineSkip(current.start);
+            if (!current.next)
+                chunks.push(codes.eos);
+            void child.write(chunks);
+        }
+        previous = current;
+        current = current.next;
+    }
+    const seen = new WeakSet();
+    let childIndex = child.events.length - 1;
+    for (let i = positions.length - 1; i >= 0; i--) {
+        ok(positions[i] !== undefined, 'expected `enter` index for linked token');
+        position = positions[i];
+        const bucket = [];
+        const linked = events[position][1];
+        ok(linked, 'expected `linked` token');
+        ok(linked.contentType, 'expected linked token with `contentType`');
+        for (let j = childIndex; j >= 0; j--) {
+            ok(child.events[j], 'expected `child.events[j]`');
+            const [event, token, self] = child.events[j];
+            if (token.start.line === token.end.line) {
+                ok(token.start.offset >= linked.start.offset, 'expected same line token to start after or at linked chunk start');
+                ok(token.end.offset <= linked.end.offset, 'expected same line token to end before or at linked chunk end');
+                bucket.unshift([event, token, self]);
+                seen.add(token);
+                continue;
+            }
+            if (token.start._bufferIndex < 0 &&
+                token.end._bufferIndex < 0 &&
+                token.end.offset !== token.start.offset) {
+                if (seen.has(token)) {
+                    ok(event === ev.enter, 'expected line ending `enter` event');
+                    bucket.unshift([event, token, self]);
+                    continue;
+                }
+                if (token.end.column !== 1) {
+                    const stream = self.sliceStream(token);
+                    ok(stream.length === 1, 'expected `token` to span one chunk');
+                    ok(stream[0] !== undefined, 'expected chunk');
+                    ok(stream[0] !== codes.eos, 'expected no end of stream code');
+                    ok(typeof stream[0] === 'number', 'expected character code chunk');
+                    token.end.column = 1;
+                    token.end.offset = eolset(stream[0], token.start.offset);
+                }
+                ok(event === ev.exit, 'expected line ending `exit` event');
+            }
+            if (event === ev.exit) {
+                if (
+                token.end.offset <= linked.start.offset ||
+                    token.end.offset > linked.end.offset) {
+                    childIndex = j;
+                    break;
+                }
+                bucket.unshift([event, token, self]);
+                seen.add(token);
+                continue;
+            }
+            ok(seen.has(token), 'expected to have seen multiline `token`');
+            bucket.unshift([event, token, self]);
+        }
+        splice(events, position, 2, bucket);
+    }
+    child.events = [];
+    return void child.events;
+}
+
+function subtokenize(events) {
+    let found = false;
+    let index = -1;
+    while (++index < events.length) {
+        ok(events[index], 'expected `events[index]`');
+        const [event, token] = events[index];
+        if (event !== ev.enter)
+            continue;
+        if (!token.contentType)
+            continue;
+        subcontent(events, index);
+        found = true;
+        index--;
+    }
+    return !found;
+}
+
+export { subcontent, subtokenize };

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@flex-development/docmark-util-subtokenize",
+  "description": "docmark utility to tokenize subtokens",
+  "version": "1.0.0-alpha.1",
+  "keywords": [
+    "docmark",
+    "tokenize",
+    "typescript",
+    "util",
+    "utility"
+  ],
+  "license": "BSD-3-Clause",
+  "homepage": "https://github.com/flex-development/docmark-util-types/tree/main/packages/flex-development/docmark-util-subtokenize",
+  "repository": {
+    "directory": "packages/docmark-util-subtokenize",
+    "type": "git",
+    "url": "git+https://github.com/flex-development/docmark.git"
+  },
+  "bugs": "https://github.com/flex-development/docmark/issues",
+  "author": {
+    "name": "Lexus Drumgold",
+    "url": "https://github.com/unicornware"
+  },
+  "publishConfig": {
+    "access": "public",
+    "diff-dst-prefix": "docmark-util-subtokenize",
+    "diff-src-prefix": "docmark-util-subtokenize",
+    "directory": "./",
+    "executableFiles": [],
+    "node-options": null,
+    "pack-destination": ".",
+    "parseable": true,
+    "prefer-dedupe": true,
+    "provenance": true,
+    "tag-version-prefix": "docmark-util-subtokenize"
+  },
+  "type": "module",
+  "files": [
+    "LICENSE.md",
+    "README.md",
+    "dist"
+  ],
+  "exports": {
+    ".": {
+      "docmark": "./src/index.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "dependencies": {
+    "@flex-development/docmark-util-symbol": "1.0.0-alpha.1",
+    "@flex-development/docmark-util-types": "1.0.0-alpha.1",
+    "@flex-development/mark-util-chunked": "1.0.0",
+    "devlop": "1.1.0"
+  },
+  "sideEffects": false,
+  "build": "524f7d653272a777cde27efaac4656fb90cd4855"
+}