inkspan 0.0.0-stage → 0.1.0-beta.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts ADDED
@@ -0,0 +1,77 @@
1
+ export interface MarkdownSectionOptions {
2
+ /** GitHub Flavored Markdown by default. */
3
+ format?: "md" | "mdx";
4
+ /** Skip headings and reference definitions inside these MDX components. */
5
+ ignoreElements?: readonly string[];
6
+ }
7
+
8
+ export interface MarkdownInlineNode {
9
+ type: string;
10
+ value?: string;
11
+ alt?: string;
12
+ identifier?: string;
13
+ label?: string;
14
+ referenceType?: string;
15
+ children?: MarkdownInlineNode[];
16
+ }
17
+
18
+ export type MarkdownOutlineNode =
19
+ | { type: "definition"; identifier: string }
20
+ | {
21
+ type: "heading";
22
+ depth: number;
23
+ position: { start: { line: number } };
24
+ children: MarkdownInlineNode[];
25
+ };
26
+
27
+ export interface MarkdownSection {
28
+ title: string;
29
+ depth: number;
30
+ /** 1-based inclusive source lines. */
31
+ startLine: number;
32
+ endLine: number;
33
+ /** Exact source slice, including the heading and nested subsections. */
34
+ content: string;
35
+ }
36
+
37
+ export class MarkdownSyntaxError extends SyntaxError {
38
+ constructor(cause: Error);
39
+ readonly code: "ERR_MARKDOWN_SYNTAX";
40
+ }
41
+
42
+ export function parseMarkdownOutline(
43
+ source: string,
44
+ options?: MarkdownSectionOptions,
45
+ ): MarkdownOutlineNode[];
46
+ export function parseMarkdownSections(
47
+ source: string,
48
+ options?: MarkdownSectionOptions,
49
+ ): MarkdownSection[];
50
+
51
+ export interface MarkdownIndexOptions {
52
+ format?: "md" | "mdx";
53
+ components?: boolean | readonly string[];
54
+ codeBlocks?: boolean;
55
+ text?: boolean | { includeCode?: boolean };
56
+ }
57
+ export interface MdxComponentSpan {
58
+ name: string;
59
+ /** Half-open UTF-16 offsets suitable for source.slice(start, end). */
60
+ start: number;
61
+ end: number;
62
+ }
63
+ export interface MarkdownCodeBlock {
64
+ start: number;
65
+ end: number;
66
+ startLine: number;
67
+ endLine: number;
68
+ language: string | null;
69
+ meta: string | null;
70
+ code: string;
71
+ }
72
+ export interface MarkdownIndex {
73
+ components: MdxComponentSpan[];
74
+ codeBlocks: MarkdownCodeBlock[];
75
+ text?: string;
76
+ }
77
+ export function indexMarkdown(source: string, options?: MarkdownIndexOptions): MarkdownIndex;
package/index.js ADDED
@@ -0,0 +1,19 @@
1
+ import * as accelerator from "#accelerator";
2
+ import {
3
+ parseMarkdownOutline as outline,
4
+ parseMarkdownSections as sections,
5
+ indexMarkdown as index,
6
+ } from "./lib/core.js";
7
+
8
+ export { MarkdownSyntaxError } from "./lib/core.js";
9
+ export function parseMarkdownOutline(source, options = {}) {
10
+ return outline(accelerator, source, options);
11
+ }
12
+
13
+ export function parseMarkdownSections(source, options) {
14
+ return sections(accelerator, source, options);
15
+ }
16
+
17
+ export function indexMarkdown(source, options = {}) {
18
+ return index(accelerator, source, options);
19
+ }
@@ -0,0 +1,43 @@
1
+ export function createAccelerator(engine, initialize = () => {}) {
2
+ let initialized = false;
3
+ function checked(result, validateImportBlock) {
4
+ if (result === undefined) return undefined;
5
+ const [value, imports] = JSON.parse(result);
6
+ return imports.every(validateImportBlock) ? value : undefined;
7
+ }
8
+
9
+ function acceleratedIndex(source, mdx, components, names, fences, validateImportBlock) {
10
+ if (!source.isWellFormed()) return undefined;
11
+ if (!initialized) {
12
+ initialize();
13
+ initialized = true;
14
+ }
15
+ return checked(
16
+ engine.structural_index(source, mdx, components, names, fences),
17
+ validateImportBlock,
18
+ );
19
+ }
20
+
21
+ function acceleratedOutline(source, mdx, ignoreElements, validateImportBlock) {
22
+ // Wasm accepts Unicode scalar values. Preserve any lone JavaScript surrogate verbatim.
23
+ if (!source.isWellFormed()) return undefined;
24
+ if (!initialized) {
25
+ // A single literal ATX heading can be returned before paying the Wasm startup cost.
26
+ const literal = /^(#{1,6}) +([A-Za-z0-9][A-Za-z0-9 -]*)$/u.exec(source);
27
+ if (literal && literal[0].length === source.length) {
28
+ return [
29
+ {
30
+ type: "heading",
31
+ depth: literal[1].length,
32
+ position: { start: { line: 1 } },
33
+ children: [{ type: "text", value: literal[2].trimEnd() }],
34
+ },
35
+ ];
36
+ }
37
+ initialize();
38
+ initialized = true;
39
+ }
40
+ return checked(engine.outline(source, mdx, ignoreElements), validateImportBlock);
41
+ }
42
+ return { acceleratedIndex, acceleratedOutline };
43
+ }
@@ -0,0 +1,22 @@
1
+ import bytes from "../generated/engine-bytes.js";
2
+ import { createEngine } from "../generated/engine-factory.js";
3
+ import { createAccelerator } from "./accelerator.js";
4
+
5
+ export async function createBrowserAccelerator() {
6
+ if (typeof globalThis.WebAssembly?.instantiate !== "function") {
7
+ throw new Error(
8
+ "WebAssembly is unavailable. Enable browser WebAssembly support or use the JavaScript parser.",
9
+ );
10
+ }
11
+ if (typeof globalThis.DecompressionStream !== "function") {
12
+ throw new Error(
13
+ "`DecompressionStream` is unavailable. Use a browser with gzip decompression support or the JavaScript parser.",
14
+ );
15
+ }
16
+ const compressed = Uint8Array.from(atob(bytes), (char) => char.charCodeAt(0));
17
+ const stream = new Blob([compressed]).stream().pipeThrough(new DecompressionStream("gzip"));
18
+ const module = await new Response(stream).arrayBuffer();
19
+ const engine = createEngine();
20
+ await engine.init({ module_or_path: module });
21
+ return createAccelerator(engine);
22
+ }
package/lib/core.js ADDED
@@ -0,0 +1,250 @@
1
+ import referenceParser from "../generated/reference.js";
2
+
3
+ function validateImportBlock(value) {
4
+ try {
5
+ const { children } = referenceParser("mdx").parse(value);
6
+ return children.length === 1 && children[0].type === "mdxjsEsm" && children[0].value === value;
7
+ } catch (error) {
8
+ if (typeof error?.reason === "string") return false;
9
+ throw error;
10
+ }
11
+ }
12
+
13
+ export class MarkdownSyntaxError extends SyntaxError {
14
+ code = "ERR_MARKDOWN_SYNTAX";
15
+
16
+ constructor(cause) {
17
+ super(cause.message, { cause });
18
+ this.name = "MarkdownSyntaxError";
19
+ }
20
+ }
21
+
22
+ function parseReference(source, format) {
23
+ try {
24
+ return referenceParser(format).parse(source);
25
+ } catch (cause) {
26
+ if (typeof cause?.reason === "string") throw new MarkdownSyntaxError(cause);
27
+ throw cause;
28
+ }
29
+ }
30
+
31
+ function referenceOutline(source, format, ignoreElements) {
32
+ const root = parseReference(source, format);
33
+ const outline = [];
34
+ const pending = [root];
35
+ while (pending.length) {
36
+ const node = pending.pop();
37
+ if (
38
+ (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") &&
39
+ ignoreElements.includes(node.name)
40
+ )
41
+ continue;
42
+ if (node.type === "heading" || node.type === "definition") {
43
+ outline.push(node);
44
+ } else if (node.children) {
45
+ for (let i = node.children.length - 1; i >= 0; i--) pending.push(node.children[i]);
46
+ }
47
+ }
48
+ return outline;
49
+ }
50
+
51
+ // Offsets differ between a UTF-8 parser and JavaScript. The public outline exposes line numbers.
52
+ function inlineNode(node) {
53
+ const result = { type: node.type };
54
+ for (const key of ["value", "alt", "identifier", "label", "referenceType"]) {
55
+ if (typeof node[key] === "string") result[key] = node[key];
56
+ }
57
+ if (node.children) result.children = node.children.map(inlineNode);
58
+ return result;
59
+ }
60
+
61
+ export function parseMarkdownOutline(accelerator, source, options = {}) {
62
+ if (typeof source !== "string") throw new TypeError("Pass a Markdown string as `source`.");
63
+ const { format = "md", ignoreElements = [] } = options;
64
+ if (format !== "md" && format !== "mdx") throw new TypeError('Set `format` to "md" or "mdx".');
65
+ if (
66
+ !Array.isArray(ignoreElements) ||
67
+ !ignoreElements.every((name) => typeof name === "string" && name.length > 0)
68
+ ) {
69
+ throw new TypeError("Pass an array of component names as `ignoreElements`.");
70
+ }
71
+ const nodes =
72
+ accelerator.acceleratedOutline(source, format === "mdx", ignoreElements, validateImportBlock) ??
73
+ referenceOutline(source, format, ignoreElements);
74
+ return nodes.map((node) =>
75
+ node.type === "definition"
76
+ ? { type: "definition", identifier: node.identifier }
77
+ : {
78
+ type: "heading",
79
+ depth: node.depth,
80
+ position: { start: { line: node.position.start.line } },
81
+ children: node.children.map(inlineNode),
82
+ },
83
+ );
84
+ }
85
+
86
+ function headingText(node) {
87
+ if (node.children) return node.children.map(headingText).join("");
88
+ if (node.type === "html") return "";
89
+ return node.alt ?? node.value ?? "";
90
+ }
91
+
92
+ export function parseMarkdownSections(accelerator, source, options) {
93
+ const outline = parseMarkdownOutline(accelerator, source, options);
94
+ const starts = [0];
95
+ for (const match of source.matchAll(/\r\n|[\r\n]/g)) starts.push(match.index + match[0].length);
96
+ const sections = [];
97
+ const open = [];
98
+ for (const node of outline) {
99
+ if (node.type !== "heading") continue;
100
+ const startLine = node.position.start.line;
101
+ while (open.length && open.at(-1).depth >= node.depth) {
102
+ open.pop().endLine = startLine - 1;
103
+ }
104
+ const section = {
105
+ title: headingText(node).replace(/\s+/gu, " ").trim(),
106
+ depth: node.depth,
107
+ startLine,
108
+ endLine: starts.length,
109
+ content: "",
110
+ };
111
+ sections.push(section);
112
+ open.push(section);
113
+ }
114
+ for (const section of sections) {
115
+ section.content = source.slice(
116
+ starts[section.startLine - 1],
117
+ starts[section.endLine] ?? source.length,
118
+ );
119
+ }
120
+ return sections;
121
+ }
122
+
123
+ function indexedText(root, includeCode) {
124
+ const chunks = [];
125
+ function visit(node) {
126
+ if (
127
+ ["mdxjsEsm", "mdxFlowExpression", "mdxTextExpression", "html", "definition", "yaml"].includes(
128
+ node.type,
129
+ )
130
+ )
131
+ return;
132
+ if (node.type === "code") {
133
+ if (includeCode) chunks.push(node.value, "\n");
134
+ return;
135
+ }
136
+ if (node.type === "break") {
137
+ chunks.push("\n");
138
+ return;
139
+ }
140
+ if (node.type === "text" || node.type === "inlineCode") {
141
+ chunks.push(node.value);
142
+ return;
143
+ }
144
+ if (node.type === "image" || node.type === "imageReference") {
145
+ chunks.push(node.alt ?? "");
146
+ return;
147
+ }
148
+ for (const child of node.children ?? []) visit(child);
149
+ if (["paragraph", "heading", "listItem", "tableRow", "blockquote"].includes(node.type))
150
+ chunks.push("\n");
151
+ if (node.type === "tableCell") chunks.push(" ");
152
+ }
153
+ visit(root);
154
+ return chunks
155
+ .join("")
156
+ .split(/[\r\n]+/)
157
+ .map((line) => line.replace(/\s+/gu, " ").trim())
158
+ .filter(Boolean)
159
+ .join("\n");
160
+ }
161
+
162
+ export function indexMarkdown(accelerator, source, options = {}) {
163
+ if (typeof source !== "string") throw new TypeError("Pass a Markdown string as `source`.");
164
+ const { format = "md", components, codeBlocks, text } = options;
165
+ if (format !== "md" && format !== "mdx") throw new TypeError('Set `format` to "md" or "mdx".');
166
+ const defaults = components === undefined && codeBlocks === undefined && text === undefined;
167
+ if (
168
+ components !== undefined &&
169
+ typeof components !== "boolean" &&
170
+ !(
171
+ Array.isArray(components) &&
172
+ Array.from(components).every((n) => typeof n === "string" && n.length > 0)
173
+ )
174
+ )
175
+ throw new TypeError("Pass component names or a boolean as `components`.");
176
+ if (codeBlocks !== undefined && typeof codeBlocks !== "boolean")
177
+ throw new TypeError("Pass a boolean as `codeBlocks`.");
178
+ if (
179
+ text !== undefined &&
180
+ typeof text !== "boolean" &&
181
+ !(
182
+ text &&
183
+ typeof text === "object" &&
184
+ !Array.isArray(text) &&
185
+ (text.includeCode === undefined || typeof text.includeCode === "boolean")
186
+ )
187
+ )
188
+ throw new TypeError("Set `text` to a boolean or an object with a boolean `includeCode`.");
189
+ const names = Array.isArray(components) ? components : [];
190
+ const wantComponents = format === "mdx" && (defaults || components === true || names.length > 0);
191
+ const wantCode = codeBlocks ?? defaults;
192
+ const wantText = text ?? defaults;
193
+ if (!wantText) {
194
+ const fast = accelerator.acceleratedIndex(
195
+ source,
196
+ format === "mdx",
197
+ wantComponents,
198
+ names,
199
+ wantCode,
200
+ validateImportBlock,
201
+ );
202
+ if (fast !== undefined) return fast;
203
+ }
204
+ const bom = source.startsWith("\uFEFF") ? 1 : 0;
205
+ const root = parseReference(source.slice(bom), format);
206
+ const result = { components: [], codeBlocks: [] };
207
+ const pending = [root];
208
+ while (pending.length) {
209
+ const node = pending.pop();
210
+ if (
211
+ wantComponents &&
212
+ (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") &&
213
+ node.name &&
214
+ (!names.length || names.includes(node.name))
215
+ ) {
216
+ result.components.push({
217
+ name: node.name,
218
+ start: node.position.start.offset + bom,
219
+ end: node.position.end.offset + bom,
220
+ });
221
+ }
222
+ if (wantCode && node.type === "code") {
223
+ const start = node.position.start.offset + bom,
224
+ end = node.position.end.offset + bom;
225
+ if (source.startsWith("```", start) || source.startsWith("~~~", start))
226
+ result.codeBlocks.push({
227
+ start,
228
+ end,
229
+ startLine: node.position.start.line,
230
+ endLine: node.position.end.line,
231
+ language: node.lang ?? null,
232
+ meta: node.meta ?? null,
233
+ code: node.value,
234
+ });
235
+ }
236
+ if (node.children)
237
+ for (let i = node.children.length - 1; i >= 0; i--) pending.push(node.children[i]);
238
+ }
239
+ if (wantText)
240
+ result.text = indexedText(root, typeof wantText === "object" && wantText.includeCode === true);
241
+ return result;
242
+ }
243
+
244
+ export function createMarkdownApi(accelerator) {
245
+ return {
246
+ parseMarkdownOutline: (source, options) => parseMarkdownOutline(accelerator, source, options),
247
+ parseMarkdownSections: (source, options) => parseMarkdownSections(accelerator, source, options),
248
+ indexMarkdown: (source, options) => indexMarkdown(accelerator, source, options),
249
+ };
250
+ }
package/lib/node.js ADDED
@@ -0,0 +1,14 @@
1
+ import { Buffer } from "node:buffer";
2
+ import { gunzipSync } from "node:zlib";
3
+ import bytes from "../generated/engine-bytes.js";
4
+ import { initSync, outline, structural_index } from "../generated/engine.js";
5
+ import { createAccelerator } from "./accelerator.js";
6
+
7
+ function initialize() {
8
+ initSync({ module: gunzipSync(Buffer.from(bytes, "base64")) });
9
+ }
10
+
11
+ export const { acceleratedIndex, acceleratedOutline } = createAccelerator(
12
+ { outline, structural_index },
13
+ initialize,
14
+ );
@@ -0,0 +1,7 @@
1
+ export function acceleratedOutline() {
2
+ return undefined;
3
+ }
4
+
5
+ export function acceleratedIndex() {
6
+ return undefined;
7
+ }
package/package.json CHANGED
@@ -1,6 +1,62 @@
1
1
  {
2
2
  "name": "inkspan",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.1.0-beta.4",
4
+ "description": "Fast Markdown sections, MDX component spans and source indexes, powered by Rust.",
5
+ "license": "MIT",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/farming-labs/inkspan.git"
9
+ },
10
+ "files": [
11
+ "index.js",
12
+ "index.d.ts",
13
+ "browser.js",
14
+ "browser.d.ts",
15
+ "lib",
16
+ "generated",
17
+ "LICENSE"
18
+ ],
19
+ "type": "module",
20
+ "sideEffects": false,
21
+ "imports": {
22
+ "#accelerator": {
23
+ "browser": "./lib/portable.js",
24
+ "workerd": "./lib/portable.js",
25
+ "edge-light": "./lib/portable.js",
26
+ "node": "./lib/node.js",
27
+ "default": "./lib/portable.js"
28
+ }
29
+ },
30
+ "exports": {
31
+ ".": {
32
+ "types": "./index.d.ts",
33
+ "import": "./index.js",
34
+ "default": "./index.js"
35
+ },
36
+ "./browser": {
37
+ "types": "./browser.d.ts",
38
+ "import": "./browser.js",
39
+ "default": "./browser.js"
40
+ }
41
+ },
42
+ "publishConfig": {
43
+ "access": "public"
44
+ },
45
+ "scripts": {
46
+ "build:wasm": "node scripts/build.mjs",
47
+ "bench": "node benchmark/corpus.mjs",
48
+ "test": "node --test && node --conditions=workerd --test",
49
+ "build:reference": "node scripts/build-reference.mjs"
50
+ },
51
+ "devDependencies": {
52
+ "esbuild": "0.28.1",
53
+ "remark-gfm": "^4.0.1",
54
+ "remark-mdx": "^3.1.1",
55
+ "remark-parse": "^11.0.0",
56
+ "unified": "^11.0.5"
57
+ },
58
+ "engines": {
59
+ "node": ">=20"
60
+ },
61
+ "packageManager": "pnpm@10.9.0"
62
+ }