@markitdownjs/core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/parser.test.d.ts +2 -0
- package/dist/__tests__/parser.test.d.ts.map +1 -0
- package/dist/__tests__/parser.test.js +251 -0
- package/dist/__tests__/parser.test.js.map +1 -0
- package/dist/parser.d.ts +18 -1
- package/dist/parser.d.ts.map +1 -1
- package/dist/parser.js +38 -4
- package/dist/parser.js.map +1 -1
- package/dist/pipeline.d.ts.map +1 -1
- package/dist/pipeline.js +12 -1
- package/dist/pipeline.js.map +1 -1
- package/package.json +13 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parser.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/parser.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { MarkItDown } from "../parser.js";
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Helpers
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
function createMockConverter(id = "mock") {
|
|
7
|
+
return {
|
|
8
|
+
id,
|
|
9
|
+
supportedMimeTypes: ["text/plain"],
|
|
10
|
+
supportedExtensions: [".txt"],
|
|
11
|
+
async canConvert(input) {
|
|
12
|
+
return input.mimeType === "text/plain";
|
|
13
|
+
},
|
|
14
|
+
async convert(_input) {
|
|
15
|
+
return {
|
|
16
|
+
markdown: "# Hello\n\nWorld",
|
|
17
|
+
metadata: { title: "Test" },
|
|
18
|
+
assets: [],
|
|
19
|
+
tables: [],
|
|
20
|
+
images: [],
|
|
21
|
+
headings: [{ level: 1, text: "Hello" }],
|
|
22
|
+
format: "markdown",
|
|
23
|
+
converterId: id,
|
|
24
|
+
stats: { startTime: 0, endTime: 0, duration: 0, inputSize: 0, outputSize: 0 },
|
|
25
|
+
ast: {
|
|
26
|
+
type: "document",
|
|
27
|
+
children: [
|
|
28
|
+
{ type: "heading", level: 1, children: [{ type: "text", value: "Hello" }] },
|
|
29
|
+
{ type: "paragraph", children: [{ type: "text", value: "World" }] },
|
|
30
|
+
],
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function createMockRenderer() {
|
|
37
|
+
return {
|
|
38
|
+
render(result, format) {
|
|
39
|
+
return `custom:${format ?? "markdown"}:${result.markdown}`;
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function createMockChunker() {
|
|
44
|
+
return {
|
|
45
|
+
chunk(_ast, _options) {
|
|
46
|
+
return [
|
|
47
|
+
{
|
|
48
|
+
id: "chunk-1",
|
|
49
|
+
content: "Chunk content",
|
|
50
|
+
metadata: {
|
|
51
|
+
chunkId: "chunk-1",
|
|
52
|
+
headingPath: [],
|
|
53
|
+
tokenCount: 3,
|
|
54
|
+
startIndex: 0,
|
|
55
|
+
endIndex: 1,
|
|
56
|
+
},
|
|
57
|
+
ast: { type: "document", children: [] },
|
|
58
|
+
},
|
|
59
|
+
];
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
// Tests
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
describe("MarkItDown", () => {
|
|
67
|
+
// -----------------------------------------------------------------------
|
|
68
|
+
// registerConverter
|
|
69
|
+
// -----------------------------------------------------------------------
|
|
70
|
+
describe("registerConverter", () => {
|
|
71
|
+
it("should register a converter through the convenience method", () => {
|
|
72
|
+
const md = new MarkItDown();
|
|
73
|
+
const converter = createMockConverter("test");
|
|
74
|
+
md.registerConverter(converter);
|
|
75
|
+
expect(md.getRegistry().get("test")).toBe(converter);
|
|
76
|
+
});
|
|
77
|
+
it("should make converter available for conversion", async () => {
|
|
78
|
+
const md = new MarkItDown();
|
|
79
|
+
md.registerConverter(createMockConverter("test"));
|
|
80
|
+
const result = await md.convert({ data: "hello", mimeType: "text/plain" });
|
|
81
|
+
expect(result.markdown).toBe("# Hello\n\nWorld");
|
|
82
|
+
expect(result.converterId).toBe("test");
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
// -----------------------------------------------------------------------
|
|
86
|
+
// registerRenderer
|
|
87
|
+
// -----------------------------------------------------------------------
|
|
88
|
+
describe("registerRenderer", () => {
|
|
89
|
+
it("should use the custom renderer in convertToJson when registered", async () => {
|
|
90
|
+
const md = new MarkItDown();
|
|
91
|
+
md.registerConverter(createMockConverter("test"));
|
|
92
|
+
md.registerRenderer(createMockRenderer());
|
|
93
|
+
const json = await md.convertToJson({ data: "hello", mimeType: "text/plain" });
|
|
94
|
+
expect(json).toContain("custom:json:");
|
|
95
|
+
});
|
|
96
|
+
it("should fall back to the default renderer when no custom renderer is set", async () => {
|
|
97
|
+
const md = new MarkItDown();
|
|
98
|
+
md.registerConverter(createMockConverter("test"));
|
|
99
|
+
const json = await md.convertToJson({ data: "hello", mimeType: "text/plain" });
|
|
100
|
+
// Default renderer outputs JSON-serialized AST
|
|
101
|
+
expect(json).toContain('"type": "document"');
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
// -----------------------------------------------------------------------
|
|
105
|
+
// registerChunker
|
|
106
|
+
// -----------------------------------------------------------------------
|
|
107
|
+
describe("registerChunker", () => {
|
|
108
|
+
it("should not chunk when chunker is registered but chunking options are absent", async () => {
|
|
109
|
+
const md = new MarkItDown();
|
|
110
|
+
md.registerConverter(createMockConverter("test"));
|
|
111
|
+
md.registerChunker(createMockChunker());
|
|
112
|
+
const result = await md.convert({ data: "hello", mimeType: "text/plain" });
|
|
113
|
+
expect(result.chunks).toBeUndefined();
|
|
114
|
+
});
|
|
115
|
+
it("should not chunk when chunking options are provided but no chunker is registered", async () => {
|
|
116
|
+
const md = new MarkItDown();
|
|
117
|
+
md.registerConverter(createMockConverter("test"));
|
|
118
|
+
const result = await md.convert({
|
|
119
|
+
data: "hello",
|
|
120
|
+
mimeType: "text/plain",
|
|
121
|
+
options: { chunking: { enabled: true, strategy: "heading" } },
|
|
122
|
+
});
|
|
123
|
+
expect(result.chunks).toBeUndefined();
|
|
124
|
+
});
|
|
125
|
+
it("should auto-chunk when chunker is registered and chunking options are provided", async () => {
|
|
126
|
+
const md = new MarkItDown();
|
|
127
|
+
md.registerConverter(createMockConverter("test"));
|
|
128
|
+
const chunker = createMockChunker();
|
|
129
|
+
const chunkSpy = vi.spyOn(chunker, "chunk");
|
|
130
|
+
md.registerChunker(chunker);
|
|
131
|
+
const result = await md.convert({
|
|
132
|
+
data: "hello",
|
|
133
|
+
mimeType: "text/plain",
|
|
134
|
+
options: { chunking: { enabled: true, strategy: "heading", maxTokens: 100 } },
|
|
135
|
+
});
|
|
136
|
+
expect(chunkSpy).toHaveBeenCalledOnce();
|
|
137
|
+
expect(result.chunks).toBeDefined();
|
|
138
|
+
expect(result.chunks).toHaveLength(1);
|
|
139
|
+
expect(result.chunks[0].content).toBe("Chunk content");
|
|
140
|
+
});
|
|
141
|
+
it("should skip chunking when the result has no AST", async () => {
|
|
142
|
+
const converterWithoutAst = {
|
|
143
|
+
...createMockConverter("no-ast"),
|
|
144
|
+
async convert(_input) {
|
|
145
|
+
return {
|
|
146
|
+
markdown: "No AST here",
|
|
147
|
+
metadata: {},
|
|
148
|
+
assets: [],
|
|
149
|
+
tables: [],
|
|
150
|
+
images: [],
|
|
151
|
+
headings: [],
|
|
152
|
+
format: "markdown",
|
|
153
|
+
converterId: "no-ast",
|
|
154
|
+
stats: { startTime: 0, endTime: 0, duration: 0, inputSize: 0, outputSize: 0 },
|
|
155
|
+
// no ast property
|
|
156
|
+
};
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
const md = new MarkItDown();
|
|
160
|
+
md.registerConverter(converterWithoutAst);
|
|
161
|
+
const chunker = createMockChunker();
|
|
162
|
+
const chunkSpy = vi.spyOn(chunker, "chunk");
|
|
163
|
+
md.registerChunker(chunker);
|
|
164
|
+
const result = await md.convert({
|
|
165
|
+
data: "hello",
|
|
166
|
+
mimeType: "text/plain",
|
|
167
|
+
options: { chunking: { enabled: true, strategy: "heading" } },
|
|
168
|
+
});
|
|
169
|
+
expect(chunkSpy).not.toHaveBeenCalled();
|
|
170
|
+
expect(result.chunks).toBeUndefined();
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
// -----------------------------------------------------------------------
|
|
174
|
+
// normalizeInput (private, tested via convert)
|
|
175
|
+
// -----------------------------------------------------------------------
|
|
176
|
+
describe("normalizeInput", () => {
|
|
177
|
+
it("should extract fileName from a string path", async () => {
|
|
178
|
+
const md = new MarkItDown();
|
|
179
|
+
// Register a converter that can detect by extension
|
|
180
|
+
const converter = {
|
|
181
|
+
id: "by-ext",
|
|
182
|
+
supportedMimeTypes: ["application/x-test"],
|
|
183
|
+
supportedExtensions: [".txt"],
|
|
184
|
+
async canConvert(input) {
|
|
185
|
+
return input.fileName?.endsWith(".txt") ?? false;
|
|
186
|
+
},
|
|
187
|
+
async convert(_input) {
|
|
188
|
+
return {
|
|
189
|
+
markdown: `Converted file: ${_input.fileName}`,
|
|
190
|
+
metadata: {},
|
|
191
|
+
assets: [],
|
|
192
|
+
tables: [],
|
|
193
|
+
images: [],
|
|
194
|
+
headings: [],
|
|
195
|
+
format: "markdown",
|
|
196
|
+
converterId: "by-ext",
|
|
197
|
+
stats: { startTime: 0, endTime: 0, duration: 0, inputSize: 0, outputSize: 0 },
|
|
198
|
+
};
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
md.registerConverter(converter);
|
|
202
|
+
const result = await md.convert("path/to/document.txt");
|
|
203
|
+
expect(result.markdown).toBe("Converted file: document.txt");
|
|
204
|
+
});
|
|
205
|
+
it("should detect MIME type from Uint8Array buffer with PDF magic bytes", async () => {
|
|
206
|
+
const md = new MarkItDown();
|
|
207
|
+
// Register a converter that only accepts PDF
|
|
208
|
+
const pdfConverter = {
|
|
209
|
+
id: "pdf-detector",
|
|
210
|
+
supportedMimeTypes: ["application/pdf"],
|
|
211
|
+
supportedExtensions: [".pdf"],
|
|
212
|
+
async canConvert(input) {
|
|
213
|
+
return input.mimeType === "application/pdf";
|
|
214
|
+
},
|
|
215
|
+
async convert(_input) {
|
|
216
|
+
return {
|
|
217
|
+
markdown: "PDF detected",
|
|
218
|
+
metadata: {},
|
|
219
|
+
assets: [],
|
|
220
|
+
tables: [],
|
|
221
|
+
images: [],
|
|
222
|
+
headings: [],
|
|
223
|
+
format: "markdown",
|
|
224
|
+
converterId: "pdf-detector",
|
|
225
|
+
stats: { startTime: 0, endTime: 0, duration: 0, inputSize: 0, outputSize: 0 },
|
|
226
|
+
};
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
md.registerConverter(pdfConverter);
|
|
230
|
+
// PDF magic bytes: %PDF
|
|
231
|
+
const pdfBuffer = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x61, 0x62, 0x63]);
|
|
232
|
+
const result = await md.convert(pdfBuffer);
|
|
233
|
+
expect(result.markdown).toBe("PDF detected");
|
|
234
|
+
});
|
|
235
|
+
it("should not detect MIME from a buffer with unknown magic bytes", async () => {
|
|
236
|
+
const md = new MarkItDown();
|
|
237
|
+
const sniffSheet = [
|
|
238
|
+
["ODS (PK)", new Uint8Array([0x50, 0x4b, 0x03, 0x04])],
|
|
239
|
+
["Random", new Uint8Array([0x00, 0x01, 0x02, 0x03])],
|
|
240
|
+
];
|
|
241
|
+
for (const [, buf] of sniffSheet) {
|
|
242
|
+
// The normalizeInput for Uint8Array paths will set the detected mimeType
|
|
243
|
+
// If mimeType is detected, the registry's canConvert will use it
|
|
244
|
+
// We just verify no error is thrown — the conversion will fail with
|
|
245
|
+
// "No converter found" since we haven't registered anything for those types
|
|
246
|
+
await expect(md.convert(buf)).rejects.toThrow("No converter found");
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
});
|
|
251
|
+
//# sourceMappingURL=parser.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parser.test.js","sourceRoot":"","sources":["../../src/__tests__/parser.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAY1C,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,EAAE,GAAG,MAAM;IACtC,OAAO;QACL,EAAE;QACF,kBAAkB,EAAE,CAAC,YAAY,CAAC;QAClC,mBAAmB,EAAE,CAAC,MAAM,CAAC;QAC7B,KAAK,CAAC,UAAU,CAAC,KAAsB;YACrC,OAAO,KAAK,CAAC,QAAQ,KAAK,YAAY,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,MAAuB;YACnC,OAAO;gBACL,QAAQ,EAAE,kBAAkB;gBAC5B,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;gBAC3B,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;gBACvC,MAAM,EAAE,UAAU;gBAClB,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;gBAC7E,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,QAAQ,EAAE;wBACR,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE;wBAC3E,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE;qBACpE;iBACoB;aACxB,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB;IACzB,OAAO;QACL,MAAM,CAAC,MAAwB,EAAE,MAAe;YAC9C,OAAO,UAAU,MAAM,IAAI,UAAU,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO;QACL,KAAK,CAAC,IAAa,EAAE,QAAyB;YAC5C,OAAO;gBACL;oBACE,EAAE,EAAE,SAAS;oBACb,OAAO,EAAE,eAAe;oBACxB,QAAQ,EAAE;wBACR,OAAO,EAAE,SAAS;wBAClB,WAAW,EAAE,EAAE;wBACf,UAAU,EAAE,CAAC;wBACb,UAAU,EAAE,CAAC;wBACb,QAAQ,EAAE,CAAC;qBACZ;oBACD,GAAG,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAwB;iBAC9D;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,QAAQ;AACR,8EAA8E;AAE9E,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,0EAA0E;IAC1E,oBAAoB;IACpB,0EAA0E;IAE1E,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9C,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,mBAAmB;IACnB,0EAA0E;IAE1E,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAClD,EAAE,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAE1C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;YAC/E,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;YACvF,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;YAC/E,+CAA+C;YAC/C,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;YAC3F,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAClD,EAAE,CAAC,eAAe,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAExC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kFAAkF,EAAE,KAAK,IAAI,EAAE;YAChG,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC9B,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;aAC9D,CAAC,CAAC;YACH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gFAAgF,EAAE,KAAK,IAAI,EAAE;YAC9F,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAElD,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAE5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC9B,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE;aAC9E,CAAC,CAAC;YAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,oBAAoB,EAAE,CAAC;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,MAAO,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,mBAAmB,GAAc;gBACrC,GAAG,mBAAmB,CAAC,QAAQ,CAAC;gBAChC,KAAK,CAAC,OAAO,CAAC,MAAuB;oBACnC,OAAO;wBACL,QAAQ,EAAE,aAAa;wBACvB,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,UAAU;wBAClB,WAAW,EAAE,QAAQ;wBACrB,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;wBAC7E,kBAAkB;qBACnB,CAAC;gBACJ,CAAC;aACF,CAAC;YAEF,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAE5B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC;gBAC9B,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;aAC9D,CAAC,CAAC;YAEH,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,0EAA0E;IAC1E,+CAA+C;IAC/C,0EAA0E;IAE1E,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,oDAAoD;YACpD,MAAM,SAAS,GAAc;gBAC3B,EAAE,EAAE,QAAQ;gBACZ,kBAAkB,EAAE,CAAC,oBAAoB,CAAC;gBAC1C,mBAAmB,EAAE,CAAC,MAAM,CAAC;gBAC7B,KAAK,CAAC,UAAU,CAAC,KAAsB;oBACrC,OAAO,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;gBACnD,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,MAAuB;oBACnC,OAAO;wBACL,QAAQ,EAAE,mBAAmB,MAAM,CAAC,QAAQ,EAAE;wBAC9C,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,UAAU;wBAClB,WAAW,EAAE,QAAQ;wBACrB,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;qBAC9E,CAAC;gBACJ,CAAC;aACF,CAAC;YACF,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEhC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,6CAA6C;YAC7C,MAAM,YAAY,GAAc;gBAC9B,EAAE,EAAE,cAAc;gBAClB,kBAAkB,EAAE,CAAC,iBAAiB,CAAC;gBACvC,mBAAmB,EAAE,CAAC,MAAM,CAAC;gBAC7B,KAAK,CAAC,UAAU,CAAC,KAAsB;oBACrC,OAAO,KAAK,CAAC,QAAQ,KAAK,iBAAiB,CAAC;gBAC9C,CAAC;gBACD,KAAK,CAAC,OAAO,CAAC,MAAuB;oBACnC,OAAO;wBACL,QAAQ,EAAE,cAAc;wBACxB,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,EAAE;wBACZ,MAAM,EAAE,UAAU;wBAClB,WAAW,EAAE,cAAc;wBAC3B,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;qBAC9E,CAAC;gBACJ,CAAC;aACF,CAAC;YACF,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAEnC,wBAAwB;YACxB,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7E,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,MAAM,EAAE,GAAG,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,UAAU,GAA2B;gBACzC,CAAC,UAAU,EAAE,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtD,CAAC,QAAQ,EAAE,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;aACrD,CAAC;YAEF,KAAK,MAAM,CAAC,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjC,yEAAyE;gBACzE,iEAAiE;gBACjE,oEAAoE;gBACpE,4EAA4E;gBAC5E,MAAM,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;YACtE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/parser.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ConversionInput, ConversionResult, ConverterRegistry } from "@markitdownjs/shared";
|
|
1
|
+
import type { ConversionInput, ConversionResult, ConverterRegistry, DocumentRenderer, Chunker, Converter } from "@markitdownjs/shared";
|
|
2
2
|
export interface MarkItDownOptions {
|
|
3
3
|
registry?: ConverterRegistry;
|
|
4
4
|
/** Auto-register all installed @markitdownjs/* converter packages. */
|
|
@@ -7,6 +7,8 @@ export interface MarkItDownOptions {
|
|
|
7
7
|
export declare class MarkItDown {
|
|
8
8
|
private pipeline;
|
|
9
9
|
private renderer;
|
|
10
|
+
private customRenderer?;
|
|
11
|
+
private chunker?;
|
|
10
12
|
constructor(options?: MarkItDownOptions);
|
|
11
13
|
/**
|
|
12
14
|
* Async factory that optionally auto-registers all installed converter packages.
|
|
@@ -17,6 +19,21 @@ export declare class MarkItDown {
|
|
|
17
19
|
static create(options?: MarkItDownOptions): Promise<MarkItDown>;
|
|
18
20
|
private _loadConverterPresets;
|
|
19
21
|
getRegistry(): ConverterRegistry;
|
|
22
|
+
/**
|
|
23
|
+
* Register a converter by delegating to the internal registry.
|
|
24
|
+
*/
|
|
25
|
+
registerConverter(converter: Converter): void;
|
|
26
|
+
/**
|
|
27
|
+
* Register a custom document renderer that overrides the default MarkdownRenderer.
|
|
28
|
+
* Accepts any object implementing the DocumentRenderer interface.
|
|
29
|
+
*/
|
|
30
|
+
registerRenderer(renderer: DocumentRenderer): void;
|
|
31
|
+
/**
|
|
32
|
+
* Register a chunker for RAG document chunking.
|
|
33
|
+
* When a chunker is registered and `chunking` options are provided to `convert()`,
|
|
34
|
+
* chunks are automatically generated after conversion.
|
|
35
|
+
*/
|
|
36
|
+
registerChunker(chunker: Chunker): void;
|
|
20
37
|
convert(input: ConversionInput | File | Blob | Uint8Array | string): Promise<ConversionResult>;
|
|
21
38
|
convertToMarkdown(input: ConversionInput | File | Blob | Uint8Array | string): Promise<string>;
|
|
22
39
|
convertToJson(input: ConversionInput | File | Blob | Uint8Array | string): Promise<string>;
|
package/dist/parser.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,OAAO,EACP,SAAS,EACV,MAAM,sBAAsB,CAAC;AAM9B,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B,sEAAsE;IACtE,MAAM,CAAC,EAAE,KAAK,CAAC;CAChB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,cAAc,CAAC,CAAmB;IAC1C,OAAO,CAAC,OAAO,CAAC,CAAU;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAMvC;;;;;OAKG;WACU,MAAM,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC;YAQvD,qBAAqB;IA0BnC,WAAW,IAAI,iBAAiB;IAIhC;;OAEG;IACH,iBAAiB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAI7C;;;OAGG;IACH,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAIlD;;;;OAIG;IACH,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAIjC,OAAO,CACX,KAAK,EAAE,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,UAAU,GAAG,MAAM,GACzD,OAAO,CAAC,gBAAgB,CAAC;IAYtB,iBAAiB,CACrB,KAAK,EAAE,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,UAAU,GAAG,MAAM,GACzD,OAAO,CAAC,MAAM,CAAC;IAKZ,aAAa,CAAC,KAAK,EAAE,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAMhG,OAAO,CAAC,cAAc;CAyBvB"}
|
package/dist/parser.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import { detectMimeTypeFromData } from "@markitdownjs/shared";
|
|
1
2
|
import { DefaultConverterRegistry } from "./registry.js";
|
|
2
3
|
import { DocumentPipeline } from "./pipeline.js";
|
|
3
4
|
import { MarkdownRenderer } from "./renderer.js";
|
|
4
5
|
export class MarkItDown {
|
|
5
6
|
pipeline;
|
|
6
7
|
renderer;
|
|
8
|
+
customRenderer;
|
|
9
|
+
chunker;
|
|
7
10
|
constructor(options) {
|
|
8
11
|
const registry = options?.registry ?? new DefaultConverterRegistry();
|
|
9
12
|
this.pipeline = new DocumentPipeline(registry);
|
|
@@ -52,9 +55,35 @@ export class MarkItDown {
|
|
|
52
55
|
getRegistry() {
|
|
53
56
|
return this.pipeline.getRegistry();
|
|
54
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Register a converter by delegating to the internal registry.
|
|
60
|
+
*/
|
|
61
|
+
registerConverter(converter) {
|
|
62
|
+
this.getRegistry().register(converter);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Register a custom document renderer that overrides the default MarkdownRenderer.
|
|
66
|
+
* Accepts any object implementing the DocumentRenderer interface.
|
|
67
|
+
*/
|
|
68
|
+
registerRenderer(renderer) {
|
|
69
|
+
this.customRenderer = renderer;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Register a chunker for RAG document chunking.
|
|
73
|
+
* When a chunker is registered and `chunking` options are provided to `convert()`,
|
|
74
|
+
* chunks are automatically generated after conversion.
|
|
75
|
+
*/
|
|
76
|
+
registerChunker(chunker) {
|
|
77
|
+
this.chunker = chunker;
|
|
78
|
+
}
|
|
55
79
|
async convert(input) {
|
|
56
80
|
const normalizedInput = this.normalizeInput(input);
|
|
57
|
-
|
|
81
|
+
const result = await this.pipeline.convert(normalizedInput);
|
|
82
|
+
// Auto-chunk if a chunker is registered and chunking options are provided.
|
|
83
|
+
if (this.chunker && result.ast && normalizedInput.options?.chunking) {
|
|
84
|
+
result.chunks = this.chunker.chunk(result.ast, normalizedInput.options.chunking);
|
|
85
|
+
}
|
|
86
|
+
return result;
|
|
58
87
|
}
|
|
59
88
|
async convertToMarkdown(input) {
|
|
60
89
|
const result = await this.convert(input);
|
|
@@ -62,14 +91,19 @@ export class MarkItDown {
|
|
|
62
91
|
}
|
|
63
92
|
async convertToJson(input) {
|
|
64
93
|
const result = await this.convert(input);
|
|
65
|
-
|
|
94
|
+
const renderer = this.customRenderer ?? this.renderer;
|
|
95
|
+
return renderer.render(result, "json");
|
|
66
96
|
}
|
|
67
97
|
normalizeInput(input) {
|
|
68
98
|
if (typeof input === "string") {
|
|
69
|
-
|
|
99
|
+
// Extract fileName from the path for extension-based MIME detection
|
|
100
|
+
const fileName = input.split(/[/\\]/).pop();
|
|
101
|
+
return { data: input, fileName };
|
|
70
102
|
}
|
|
71
103
|
if (input instanceof Uint8Array) {
|
|
72
|
-
|
|
104
|
+
// Auto-detect MIME from buffer magic bytes when no mimeType is provided
|
|
105
|
+
const mimeType = detectMimeTypeFromData(input);
|
|
106
|
+
return { data: input, mimeType };
|
|
73
107
|
}
|
|
74
108
|
if (typeof File !== "undefined" && input instanceof File) {
|
|
75
109
|
return {
|
package/dist/parser.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAQjD,MAAM,OAAO,UAAU;IACb,QAAQ,CAAmB;IAC3B,QAAQ,CAAmB;IAC3B,cAAc,CAAoB;IAClC,OAAO,CAAW;IAE1B,YAAY,OAA2B;QACrC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,IAAI,wBAAwB,EAAE,CAAC;QACrE,IAAI,CAAC,QAAQ,GAAG,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACzC,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAA2B;QAC7C,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,OAAO,EAAE,MAAM,KAAK,KAAK,EAAE,CAAC;YAC9B,MAAM,QAAQ,CAAC,qBAAqB,EAAE,CAAC;QACzC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,MAAM,IAAI,GAAuB;YAC/B,CAAC,mBAAmB,EAAE,cAAc,CAAC;YACrC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,mBAAmB,EAAE,cAAc,CAAC;YACrC,CAAC,oBAAoB,EAAE,eAAe,CAAC;YACvC,CAAC,mBAAmB,EAAE,cAAc,CAAC;YACrC,CAAC,uBAAuB,EAAE,kBAAkB,CAAC;YAC7C,CAAC,yBAAyB,EAAE,cAAc,CAAC;YAC3C,CAAC,qBAAqB,EAAE,gBAAgB,CAAC;SAC1C,CAAC;QACF,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAkC,CAAC;gBACjE,IAAI,GAAG,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,CAAE,EAAE,CAAC,CAAC;YAC7D,CAAC;YAAC,MAAM,CAAC;gBACP,yCAAyC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,SAAoB;QACpC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,QAA0B;QACzC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,OAAgB;QAC9B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CACX,KAA0D;QAE1D,MAAM,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAE5D,2EAA2E;QAC3E,IAAI,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,IAAI,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC;YACpE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACnF,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,KAA0D;QAE1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAA0D;QAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC;QACtD,OAAO,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAEO,cAAc,CACpB,KAA0D;QAE1D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,oEAAoE;YACpE,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;YAC5C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACnC,CAAC;QACD,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YAChC,wEAAwE;YACxE,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC/C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACnC,CAAC;QACD,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;YACzD,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,KAAK,CAAC,IAAI;gBACpB,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,SAAS;aAClC,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;YACzD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,KAAwB,CAAC;IAClC,CAAC;CACF"}
|
package/dist/pipeline.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAEhB,iBAAiB,EAClB,MAAM,sBAAsB,CAAC;AAK9B,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAoB;gBAExB,QAAQ,CAAC,EAAE,iBAAiB;IAIxC,WAAW,IAAI,iBAAiB;IAI1B,OAAO,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2E1D,mBAAmB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAW7E"}
|
package/dist/pipeline.js
CHANGED
|
@@ -39,7 +39,18 @@ export class DocumentPipeline {
|
|
|
39
39
|
catch (error) {
|
|
40
40
|
if (error instanceof CancellationError)
|
|
41
41
|
throw error;
|
|
42
|
-
|
|
42
|
+
// Check if the converter set any partial result/warnings before throwing
|
|
43
|
+
const partialResult = error.partialResult;
|
|
44
|
+
const partialWarnings = error.warnings;
|
|
45
|
+
if (partialResult) {
|
|
46
|
+
result = partialResult;
|
|
47
|
+
if (partialWarnings) {
|
|
48
|
+
result.warnings = [...(result.warnings ?? []), ...partialWarnings];
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
throw new ConversionError(`Converter "${converter.id}" failed: ${error instanceof Error ? error.message : String(error)}`, converter.id, error instanceof Error ? error : undefined);
|
|
53
|
+
}
|
|
43
54
|
}
|
|
44
55
|
for (const middleware of middlewares) {
|
|
45
56
|
if (middleware.afterConvert) {
|
package/dist/pipeline.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAEzD,MAAM,OAAO,gBAAgB;IACnB,QAAQ,CAAoB;IAEpC,YAAY,QAA4B;QACtC,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,wBAAwB,EAAE,CAAC;IAC7D,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAsB;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC;QACrC,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7C,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpB,MAAM,aAAa,GAAoB;YACrC,GAAG,KAAK;YACR,IAAI;SACL,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,eAAe,CACvB,+BAA+B,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9E,CAAC;QACJ,CAAC;QAED,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpB,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;QACnD,IAAI,cAAc,GAAG,aAAa,CAAC;QAEnC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,UAAU,CAAC,aAAa,EAAE,CAAC;gBAC7B,cAAc,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,WAAW,CAAC,MAAM,CAAC,CAAC;QAEpB,IAAI,MAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,iBAAiB;gBAAE,MAAM,KAAK,CAAC;YACpD,yEAAyE;YACzE,MAAM,aAAa,GAAI,KAAa,CAAC,aAA6C,CAAC;YACnF,MAAM,eAAe,GAAI,KAAa,CAAC,QAA2C,CAAC;YACnF,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,GAAG,aAAa,CAAC;gBACvB,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,GAAG,eAAe,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,eAAe,CACvB,cAAc,SAAS,CAAC,EAAE,aAAa,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC/F,SAAS,CAAC,EAAE,EACZ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAC3C,CAAC;YACJ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;gBAC5B,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,CAAC,KAAK,GAAG;YACb,GAAG,MAAM,CAAC,KAAK;YACf,SAAS;YACT,OAAO;YACP,QAAQ,EAAE,OAAO,GAAG,SAAS;YAC7B,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;SACnC,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAsB;QAC9C,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,MAAM,aAAa,GAAG,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;gBACxD,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC3C,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@markitdownjs/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Core document conversion pipeline for MarkItDownJS",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -17,12 +17,22 @@
|
|
|
17
17
|
"dist"
|
|
18
18
|
],
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@markitdownjs/shared": "0.
|
|
20
|
+
"@markitdownjs/shared": "0.4.0"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
23
|
"typescript": "^5.5.0",
|
|
24
|
-
"vitest": "^4.1.
|
|
24
|
+
"vitest": "^4.1.9"
|
|
25
25
|
},
|
|
26
|
+
"keywords": [
|
|
27
|
+
"converter",
|
|
28
|
+
"document-conversion",
|
|
29
|
+
"markdown",
|
|
30
|
+
"markitdownjs",
|
|
31
|
+
"parser",
|
|
32
|
+
"pipeline",
|
|
33
|
+
"registry",
|
|
34
|
+
"typescript"
|
|
35
|
+
],
|
|
26
36
|
"scripts": {
|
|
27
37
|
"build": "tsc --project tsconfig.json",
|
|
28
38
|
"dev": "tsc --watch --project tsconfig.json",
|