mdream 2.0.0-beta.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -5,8 +5,6 @@ export interface CleanOptions {
5
5
  fragments?: boolean;
6
6
  /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
7
7
  emptyLinks?: boolean;
8
- /** Collapse 3+ consecutive blank lines to 2 */
9
- blankLines?: boolean;
10
8
  /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
11
9
  redundantLinks?: boolean;
12
10
  /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
package/dist/index.mjs CHANGED
@@ -22,19 +22,38 @@ const CLEAN_ALL = {
22
22
  emptyImages: true,
23
23
  emptyLinkText: true
24
24
  };
25
- function resolveCleanConfig(options, minimal) {
26
- const cleanUrls = options.cleanUrls === true;
27
- let cleanOpt = options.clean;
28
- if (cleanOpt === void 0) {
29
- if (!minimal) return { cleanUrls };
30
- cleanOpt = true;
25
+ const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
26
+ function unknownOptionMessage(key, value) {
27
+ switch (key) {
28
+ case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
29
+ case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
30
+ case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
31
31
  }
32
- if (!cleanOpt) return { cleanUrls };
33
- const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
34
- return {
35
- cleanUrls: cleanUrls || resolved.urls === true,
36
- clean: resolved
37
- };
32
+ return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
33
+ }
34
+ function assertKnownOptions(options) {
35
+ for (const key in options) {
36
+ switch (key) {
37
+ case "origin":
38
+ case "clean":
39
+ case "minimal":
40
+ case "frontmatter":
41
+ case "isolateMain":
42
+ case "tailwind":
43
+ case "filter":
44
+ case "extraction":
45
+ case "tagOverrides":
46
+ case "wrapWidth":
47
+ case "format": continue;
48
+ }
49
+ const value = options[key];
50
+ if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
51
+ }
52
+ }
53
+ function resolveClean(clean, minimal) {
54
+ if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
55
+ if (!clean) return void 0;
56
+ return clean === true ? CLEAN_ALL : clean;
38
57
  }
39
58
  function resolveFrontmatter(opt) {
40
59
  if (typeof opt === "function") return {
@@ -51,24 +70,18 @@ function resolveFrontmatter(opt) {
51
70
  return { config: {} };
52
71
  }
53
72
  function resolveOptions(options) {
54
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
73
+ assertKnownOptions(options);
55
74
  const minimal = options.minimal === true;
56
- const plugins = options.plugins ? { ...options.plugins } : {};
75
+ const plugins = {};
57
76
  let frontmatterCallback;
58
- const frontmatterDisabled = options.frontmatter === false;
59
- if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
77
+ if (minimal ? options.frontmatter !== false : options.frontmatter) {
60
78
  const fm = resolveFrontmatter(options.frontmatter);
61
79
  plugins.frontmatter = fm.config;
62
80
  frontmatterCallback = fm.callback;
63
- } else if (frontmatterDisabled) delete plugins.frontmatter;
64
- const isolateMainDisabled = options.isolateMain === false;
65
- if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
66
- else if (isolateMainDisabled) delete plugins.isolateMain;
67
- const tailwindDisabled = options.tailwind === false;
68
- if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
69
- else if (tailwindDisabled) delete plugins.tailwind;
70
- if (options.filter === false) delete plugins.filter;
71
- else if (minimal) plugins.filter = {
81
+ }
82
+ if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
83
+ if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
84
+ if (minimal && options.filter !== false) plugins.filter = {
72
85
  ...options.filter,
73
86
  exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
74
87
  };
@@ -88,12 +101,10 @@ function resolveOptions(options) {
88
101
  }
89
102
  plugins.tagOverrides = overrides;
90
103
  }
91
- const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
92
104
  return {
93
105
  napiOpts: {
94
106
  origin: options.origin,
95
- cleanUrls,
96
- clean,
107
+ clean: resolveClean(options.clean, minimal),
97
108
  plugins,
98
109
  wrapWidth: options.wrapWidth,
99
110
  format: options.format
@@ -115,9 +126,9 @@ function convertResult(convert, html, options = {}) {
115
126
  deliverPluginData(result, resolved);
116
127
  return result;
117
128
  }
118
- async function* pumpStream(stream, htmlStream, callbacks, { decodeInJs = false, mapError = (error) => error } = {}) {
129
+ async function* pumpStream(stream, htmlStream, callbacks, { mapError = (error) => error } = {}) {
119
130
  const reader = htmlStream.getReader();
120
- const decoder = decodeInJs ? new TextDecoder() : void 0;
131
+ const decoder = stream.processChunkBytes ? void 0 : new TextDecoder();
121
132
  try {
122
133
  while (true) {
123
134
  const { done, value } = await reader.read();
@@ -173,12 +184,11 @@ async function* streamHtmlToMarkdown(htmlStream, options = {}) {
173
184
  const stream = new MarkdownStream(resolved.napiOpts);
174
185
  yield* pumpStream({
175
186
  processChunk: (chunk) => stream.processChunk(carry.take(chunk)),
176
- processChunkBytes: (chunk) => stream.processChunkBytes(chunk),
177
187
  finish: () => {
178
188
  const held = carry.flush();
179
189
  return held ? stream.processChunk(held) + stream.finish() : stream.finish();
180
190
  },
181
191
  takeData: () => stream.takeData()
182
- }, htmlStream, resolved, { decodeInJs: true });
192
+ }, htmlStream, resolved);
183
193
  }
184
194
  export { htmlToMarkdown, streamHtmlToMarkdown };
package/dist/wasm.d.mts CHANGED
@@ -1,6 +1,5 @@
1
- import mdream_edge_default, { MarkdownStream as MarkdownStream$1 } from "../wasm/mdream_edge.js";
1
+ import mdream_edge_default, { InitInput, InitOutput, MarkdownStream as MarkdownStream$1, SyncInitInput, initSync } from "../wasm/mdream_edge.js";
2
2
  import { MdreamNapiResult } from "../napi/index.js";
3
- export * from "../wasm/mdream_edge.js";
4
3
  interface CleanOptions {
5
4
  /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
6
5
  urls?: boolean;
@@ -8,8 +7,6 @@ interface CleanOptions {
8
7
  fragments?: boolean;
9
8
  /** Strip links with meaningless hrefs (#, javascript:void(0)) → plain text */
10
9
  emptyLinks?: boolean;
11
- /** Collapse 3+ consecutive blank lines to 2 */
12
- blankLines?: boolean;
13
10
  /** Strip links where text equals URL: [https://x.com](https://x.com) → https://x.com */
14
11
  redundantLinks?: boolean;
15
12
  /** Strip self-referencing heading anchors: ## [Title](#title) → ## Title */
@@ -97,4 +94,4 @@ export declare class MarkdownStream extends MarkdownStream$1 {
97
94
  constructor(options?: Partial<MdreamOptions>);
98
95
  finish(): string;
99
96
  }
100
- export { mdream_edge_default as default };
97
+ export { type InitInput, type InitOutput, type SyncInitInput, mdream_edge_default as default, initSync };
package/dist/wasm.mjs CHANGED
@@ -1,5 +1,4 @@
1
- import mdream_edge_default, { MarkdownStream as MarkdownStream$1, htmlToMarkdownBytes as htmlToMarkdownBytes$1, htmlToMarkdownResult as htmlToMarkdownResult$1 } from "../wasm/mdream_edge.js";
2
- export * from "../wasm/mdream_edge.js";
1
+ import mdream_edge_default, { MarkdownStream as MarkdownStream$1, htmlToMarkdownBytes as htmlToMarkdownBytes$1, htmlToMarkdownResult as htmlToMarkdownResult$1, initSync } from "../wasm/mdream_edge.js";
3
2
  const MINIMAL_FILTER_EXCLUDE = [
4
3
  "form",
5
4
  "fieldset",
@@ -23,19 +22,38 @@ const CLEAN_ALL = {
23
22
  emptyImages: true,
24
23
  emptyLinkText: true
25
24
  };
26
- function resolveCleanConfig(options, minimal) {
27
- const cleanUrls = options.cleanUrls === true;
28
- let cleanOpt = options.clean;
29
- if (cleanOpt === void 0) {
30
- if (!minimal) return { cleanUrls };
31
- cleanOpt = true;
25
+ const VALID_OPTIONS = "origin, clean, minimal, frontmatter, isolateMain, tailwind, filter, extraction, tagOverrides, wrapWidth, format";
26
+ function unknownOptionMessage(key, value) {
27
+ switch (key) {
28
+ case "preset": return "mdream has no `preset` option. Pass { minimal: true }.";
29
+ case "cleanUrls": return "mdream has no `cleanUrls` option. Pass { clean: { urls: true } }.";
30
+ case "plugins": return Array.isArray(value) ? "Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://github.com/harlan-zw/mdream/tree/main/packages/js#migrating-from-v1" : "mdream takes plugin options at the top level, such as { frontmatter: true }.";
32
31
  }
33
- if (!cleanOpt) return { cleanUrls };
34
- const resolved = cleanOpt === true ? CLEAN_ALL : cleanOpt;
35
- return {
36
- cleanUrls: cleanUrls || resolved.urls === true,
37
- clean: resolved
38
- };
32
+ return `mdream has no \`${key}\` option. Valid options: ${VALID_OPTIONS}.`;
33
+ }
34
+ function assertKnownOptions(options) {
35
+ for (const key in options) {
36
+ switch (key) {
37
+ case "origin":
38
+ case "clean":
39
+ case "minimal":
40
+ case "frontmatter":
41
+ case "isolateMain":
42
+ case "tailwind":
43
+ case "filter":
44
+ case "extraction":
45
+ case "tagOverrides":
46
+ case "wrapWidth":
47
+ case "format": continue;
48
+ }
49
+ const value = options[key];
50
+ if (value !== void 0) throw new TypeError(unknownOptionMessage(key, value));
51
+ }
52
+ }
53
+ function resolveClean(clean, minimal) {
54
+ if (clean === void 0) return minimal ? CLEAN_ALL : void 0;
55
+ if (!clean) return void 0;
56
+ return clean === true ? CLEAN_ALL : clean;
39
57
  }
40
58
  function resolveFrontmatter(opt) {
41
59
  if (typeof opt === "function") return {
@@ -52,24 +70,18 @@ function resolveFrontmatter(opt) {
52
70
  return { config: {} };
53
71
  }
54
72
  function resolveOptions(options) {
55
- if (Array.isArray(options.plugins)) throw new TypeError("Custom hook plugins require @mdream/js. Pass declarative config (e.g. { frontmatter: true }) to the Rust engine, or import { htmlToMarkdown } from '@mdream/js' for hook-based plugins. See https://mdream.dev/v1-migration#custom-plugins");
73
+ assertKnownOptions(options);
56
74
  const minimal = options.minimal === true;
57
- const plugins = options.plugins ? { ...options.plugins } : {};
75
+ const plugins = {};
58
76
  let frontmatterCallback;
59
- const frontmatterDisabled = options.frontmatter === false;
60
- if (minimal ? !frontmatterDisabled : !!options.frontmatter) {
77
+ if (minimal ? options.frontmatter !== false : options.frontmatter) {
61
78
  const fm = resolveFrontmatter(options.frontmatter);
62
79
  plugins.frontmatter = fm.config;
63
80
  frontmatterCallback = fm.callback;
64
- } else if (frontmatterDisabled) delete plugins.frontmatter;
65
- const isolateMainDisabled = options.isolateMain === false;
66
- if (minimal ? !isolateMainDisabled : options.isolateMain) plugins.isolateMain = true;
67
- else if (isolateMainDisabled) delete plugins.isolateMain;
68
- const tailwindDisabled = options.tailwind === false;
69
- if (minimal ? !tailwindDisabled : options.tailwind) plugins.tailwind = true;
70
- else if (tailwindDisabled) delete plugins.tailwind;
71
- if (options.filter === false) delete plugins.filter;
72
- else if (minimal) plugins.filter = {
81
+ }
82
+ if (minimal ? options.isolateMain !== false : options.isolateMain) plugins.isolateMain = true;
83
+ if (minimal ? options.tailwind !== false : options.tailwind) plugins.tailwind = true;
84
+ if (minimal && options.filter !== false) plugins.filter = {
73
85
  ...options.filter,
74
86
  exclude: [...MINIMAL_FILTER_EXCLUDE, ...options.filter?.exclude ?? []]
75
87
  };
@@ -89,12 +101,10 @@ function resolveOptions(options) {
89
101
  }
90
102
  plugins.tagOverrides = overrides;
91
103
  }
92
- const { cleanUrls, clean } = resolveCleanConfig(options, minimal);
93
104
  return {
94
105
  napiOpts: {
95
106
  origin: options.origin,
96
- cleanUrls,
97
- clean,
107
+ clean: resolveClean(options.clean, minimal),
98
108
  plugins,
99
109
  wrapWidth: options.wrapWidth,
100
110
  format: options.format
@@ -137,4 +147,4 @@ var MarkdownStream = class extends MarkdownStream$1 {
137
147
  return markdown;
138
148
  }
139
149
  };
140
- export { MarkdownStream, mdream_edge_default as default, htmlToMarkdown, htmlToMarkdownBytes, htmlToMarkdownResult };
150
+ export { MarkdownStream, mdream_edge_default as default, htmlToMarkdown, htmlToMarkdownBytes, htmlToMarkdownResult, initSync };
package/napi/index.d.mts CHANGED
@@ -12,7 +12,6 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-was
12
12
  export declare class MarkdownStream {
13
13
  constructor(options?: HtmlToMarkdownOptions | undefined | null)
14
14
  processChunk(chunk: string): string
15
- processChunkBytes(chunk: Uint8Array): string
16
15
  finish(): string
17
16
  /**
18
17
  * Frontmatter and extracted elements collected so far. Call after
@@ -21,22 +20,10 @@ export declare class MarkdownStream {
21
20
  takeData(): MdreamStreamData
22
21
  }
23
22
 
24
- export interface ChunkLocNapi {
25
- from: number
26
- to: number
27
- }
28
-
29
- export interface ChunkMetadataNapi {
30
- headers?: Record<string, string>
31
- code?: string
32
- loc?: ChunkLocNapi
33
- }
34
-
35
23
  export interface CleanOptionsNapi {
36
24
  urls?: boolean
37
25
  fragments?: boolean
38
26
  emptyLinks?: boolean
39
- blankLines?: boolean
40
27
  redundantLinks?: boolean
41
28
  selfLinkHeadings?: boolean
42
29
  emptyImages?: boolean
@@ -67,24 +54,14 @@ export interface FrontmatterOptions {
67
54
 
68
55
  export declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
69
56
 
70
- export declare function htmlToMarkdownBytes(html: Uint8Array, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
71
-
72
- export declare function htmlToMarkdownChunks(html: string, options?: HtmlToMarkdownOptions | undefined | null, splitterOptions?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
73
-
74
57
  export interface HtmlToMarkdownOptions {
75
58
  origin?: string
76
- cleanUrls?: boolean
77
59
  clean?: CleanOptionsNapi
78
60
  plugins?: PluginOptions
79
61
  wrapWidth?: number
80
62
  format?: "markdown" | "text" | "html"
81
63
  }
82
64
 
83
- export interface MarkdownChunkNapi {
84
- content: string
85
- metadata: ChunkMetadataNapi
86
- }
87
-
88
65
  export interface MdreamNapiResult {
89
66
  markdown: string
90
67
  extracted?: Array<ExtractedElementNapi>
@@ -106,16 +83,6 @@ export interface PluginOptions {
106
83
  tagOverrides?: Record<string, TagOverrideNapi>
107
84
  }
108
85
 
109
- export declare function splitMarkdown(markdown: string, options?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
110
-
111
- export interface SplitterOptionsNapi {
112
- headersToSplitOn?: Array<number>
113
- returnEachLine?: boolean
114
- stripHeaders?: boolean
115
- chunkSize?: number
116
- chunkOverlap?: number
117
- }
118
-
119
86
  export interface TagOverrideNapi {
120
87
  enter?: string
121
88
  exit?: string
package/napi/index.d.ts CHANGED
@@ -12,7 +12,6 @@ export declare const __napiBindingTarget: 'native' | 'wasm32-wasi' | 'wasm32-was
12
12
  export declare class MarkdownStream {
13
13
  constructor(options?: HtmlToMarkdownOptions | undefined | null)
14
14
  processChunk(chunk: string): string
15
- processChunkBytes(chunk: Uint8Array): string
16
15
  finish(): string
17
16
  /**
18
17
  * Frontmatter and extracted elements collected so far. Call after
@@ -21,22 +20,10 @@ export declare class MarkdownStream {
21
20
  takeData(): MdreamStreamData
22
21
  }
23
22
 
24
- export interface ChunkLocNapi {
25
- from: number
26
- to: number
27
- }
28
-
29
- export interface ChunkMetadataNapi {
30
- headers?: Record<string, string>
31
- code?: string
32
- loc?: ChunkLocNapi
33
- }
34
-
35
23
  export interface CleanOptionsNapi {
36
24
  urls?: boolean
37
25
  fragments?: boolean
38
26
  emptyLinks?: boolean
39
- blankLines?: boolean
40
27
  redundantLinks?: boolean
41
28
  selfLinkHeadings?: boolean
42
29
  emptyImages?: boolean
@@ -67,24 +54,14 @@ export interface FrontmatterOptions {
67
54
 
68
55
  export declare function htmlToMarkdown(html: string, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
69
56
 
70
- export declare function htmlToMarkdownBytes(html: Uint8Array, options?: HtmlToMarkdownOptions | undefined | null): MdreamNapiResult
71
-
72
- export declare function htmlToMarkdownChunks(html: string, options?: HtmlToMarkdownOptions | undefined | null, splitterOptions?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
73
-
74
57
  export interface HtmlToMarkdownOptions {
75
58
  origin?: string
76
- cleanUrls?: boolean
77
59
  clean?: CleanOptionsNapi
78
60
  plugins?: PluginOptions
79
61
  wrapWidth?: number
80
62
  format?: "markdown" | "text" | "html"
81
63
  }
82
64
 
83
- export interface MarkdownChunkNapi {
84
- content: string
85
- metadata: ChunkMetadataNapi
86
- }
87
-
88
65
  export interface MdreamNapiResult {
89
66
  markdown: string
90
67
  extracted?: Array<ExtractedElementNapi>
@@ -106,16 +83,6 @@ export interface PluginOptions {
106
83
  tagOverrides?: Record<string, TagOverrideNapi>
107
84
  }
108
85
 
109
- export declare function splitMarkdown(markdown: string, options?: SplitterOptionsNapi | undefined | null): Array<MarkdownChunkNapi>
110
-
111
- export interface SplitterOptionsNapi {
112
- headersToSplitOn?: Array<number>
113
- returnEachLine?: boolean
114
- stripHeaders?: boolean
115
- chunkSize?: number
116
- chunkOverlap?: number
117
- }
118
-
119
86
  export interface TagOverrideNapi {
120
87
  enter?: string
121
88
  exit?: string