opencode-rag-plugin 2.5.0 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,15 @@
1
- /**
2
- * @fileoverview Text extraction and paragraph-based chunker for Word (.docx) documents using mammoth.
3
- */
4
1
  import type { Chunker, Chunk } from "../core/interfaces.js";
5
2
  /**
6
3
  * Extract the raw text from a Word (.docx) file buffer.
7
- * Uses the `mammoth` library under the hood.
4
+ *
5
+ * Unzips the archive and reads `word/document.xml` directly. Only that single
6
+ * entry is decompressed, and an inflated-size cap is enforced, so a crafted
7
+ * archive cannot exhaust memory.
8
8
  * @param buffer - Raw buffer of the .docx file.
9
9
  * @returns The extracted raw text as a string.
10
+ * @throws If the buffer is not a valid ZIP archive or lacks `word/document.xml`.
10
11
  */
11
- export declare function extractDocxText(buffer: Buffer): Promise<string>;
12
+ export declare function extractDocxText(buffer: Buffer): string;
12
13
  /**
13
14
  * Chunker for Word (.docx) documents.
14
15
  * Splits the extracted plain text by paragraph boundaries, grouping small
@@ -1,17 +1,129 @@
1
+ /**
2
+ * @fileoverview Text extraction and paragraph-based chunker for Word (.docx) documents.
3
+ *
4
+ * A `.docx` file is a ZIP archive containing `word/document.xml`, where the text
5
+ * lives in `<w:t>` runs and paragraph boundaries are `<w:p>` elements. Extraction
6
+ * is done directly with fflate's `unzipSync` so the plugin pulls in no document
7
+ * parser (the previous `mammoth` dependency carried six transitive packages,
8
+ * several of them with known prototype-pollution advisories).
9
+ */
10
+ import { unzipSync } from "fflate";
1
11
  import { uuid } from "./uuid.js";
2
12
  const MAX_CHUNK_CHARS = 4000;
3
13
  const MIN_GROUP_CHARS = 300;
4
14
  const PARAGRAPH_SPLIT = /\n\s*\n/;
15
+ /** ZIP entry holding the document body. */
16
+ const DOCUMENT_XML = "word/document.xml";
17
+ /**
18
+ * Upper bound on the decompressed size of `word/document.xml`.
19
+ *
20
+ * `.docx` files are frequently untrusted, and a small archive can inflate to
21
+ * gigabytes (a zip bomb). Anything past this is refused instead of buffered.
22
+ */
23
+ const MAX_DOCUMENT_XML_BYTES = 64 * 1024 * 1024;
24
+ /**
25
+ * Decode the XML entities Word emits inside text runs.
26
+ * @param text - Raw (escaped) text from a `<w:t>` element.
27
+ * @returns The unescaped text.
28
+ */
29
+ function decodeXmlEntities(text) {
30
+ return text
31
+ .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16)))
32
+ .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10)))
33
+ .replace(/&lt;/g, "<")
34
+ .replace(/&gt;/g, ">")
35
+ .replace(/&quot;/g, '"')
36
+ .replace(/&apos;/g, "'")
37
+ .replace(/&amp;/g, "&");
38
+ }
39
+ /**
40
+ * Convert raw `word/document.xml` markup into plain text.
41
+ *
42
+ * Mirrors what `mammoth.extractRawText` produced for the chunker's purposes:
43
+ * `<w:t>` text is concatenated, `<w:tab>` becomes a tab, `<w:br>` becomes a
44
+ * newline, and each paragraph is terminated by a blank line so the
45
+ * {@link PARAGRAPH_SPLIT} regex still sees paragraph boundaries.
46
+ * @param xml - Contents of `word/document.xml`.
47
+ * @returns Plain text with paragraphs separated by blank lines.
48
+ */
49
+ function documentXmlToText(xml) {
50
+ let out = "";
51
+ // Text outside a <w:p> is document metadata, not body text.
52
+ let inParagraph = false;
53
+ const OPEN_TAG = /<w:(t|tab|br|p)(?:\s[^>]*)?\/?>/g;
54
+ const CLOSE_P = /<\/w:p>/g;
55
+ // Walk both matchers in document order.
56
+ let open = OPEN_TAG.exec(xml);
57
+ let close = CLOSE_P.exec(xml);
58
+ while (open !== null) {
59
+ // A closing </w:p> before the next opening tag wins.
60
+ if (close !== null && close.index < open.index) {
61
+ if (inParagraph) {
62
+ out += "\n\n";
63
+ inParagraph = false;
64
+ }
65
+ close = CLOSE_P.exec(xml);
66
+ continue;
67
+ }
68
+ const [full, name] = open;
69
+ const selfClosing = full.endsWith("/>");
70
+ if (name === "p") {
71
+ if (!selfClosing && !inParagraph) {
72
+ inParagraph = true;
73
+ }
74
+ }
75
+ else if (inParagraph) {
76
+ if (name === "tab") {
77
+ out += "\t";
78
+ }
79
+ else if (name === "br") {
80
+ out += "\n";
81
+ }
82
+ else if (!selfClosing) {
83
+ const endTag = `</w:t>`;
84
+ const end = xml.indexOf(endTag, open.index + full.length);
85
+ const stop = end === -1 ? xml.length : end;
86
+ out += decodeXmlEntities(xml.slice(open.index + full.length, stop));
87
+ }
88
+ }
89
+ open = OPEN_TAG.exec(xml);
90
+ }
91
+ // Unterminated trailing paragraph.
92
+ if (inParagraph)
93
+ out += "\n\n";
94
+ return out;
95
+ }
5
96
  /**
6
97
  * Extract the raw text from a Word (.docx) file buffer.
7
- * Uses the `mammoth` library under the hood.
98
+ *
99
+ * Unzips the archive and reads `word/document.xml` directly. Only that single
100
+ * entry is decompressed, and an inflated-size cap is enforced, so a crafted
101
+ * archive cannot exhaust memory.
8
102
  * @param buffer - Raw buffer of the .docx file.
9
103
  * @returns The extracted raw text as a string.
104
+ * @throws If the buffer is not a valid ZIP archive or lacks `word/document.xml`.
10
105
  */
11
- export async function extractDocxText(buffer) {
12
- const mammoth = await import("mammoth");
13
- const result = await mammoth.extractRawText({ buffer });
14
- return result.value;
106
+ export function extractDocxText(buffer) {
107
+ let tooLarge = false;
108
+ const entries = unzipSync(new Uint8Array(buffer), {
109
+ filter: (file) => {
110
+ if (file.name !== DOCUMENT_XML)
111
+ return false;
112
+ if (file.originalSize > MAX_DOCUMENT_XML_BYTES) {
113
+ tooLarge = true;
114
+ return false;
115
+ }
116
+ return true;
117
+ },
118
+ });
119
+ if (tooLarge) {
120
+ throw new Error(`DOCX too large: "${DOCUMENT_XML}" exceeds ${MAX_DOCUMENT_XML_BYTES} bytes when decompressed`);
121
+ }
122
+ const doc = entries[DOCUMENT_XML];
123
+ if (!doc) {
124
+ throw new Error(`Not a valid DOCX file: missing "${DOCUMENT_XML}"`);
125
+ }
126
+ return documentXmlToText(Buffer.from(doc).toString("utf8"));
15
127
  }
16
128
  /**
17
129
  * Chunker for Word (.docx) documents.
@@ -5,7 +5,7 @@ import type { ExtractResult } from "./types.js";
5
5
  /** Recognized DOCX file extension set. */
6
6
  export declare const DOCX_EXTENSIONS: Set<string>;
7
7
  /**
8
- * Extract text content from a DOCX buffer using mammoth.
8
+ * Extract text content from a DOCX buffer.
9
9
  * @param _filePath - Unused, retained for API consistency.
10
10
  * @param buffer - Raw DOCX file bytes.
11
11
  */
@@ -4,14 +4,14 @@
4
4
  /** Recognized DOCX file extension set. */
5
5
  export const DOCX_EXTENSIONS = new Set([".docx"]);
6
6
  /**
7
- * Extract text content from a DOCX buffer using mammoth.
7
+ * Extract text content from a DOCX buffer.
8
8
  * @param _filePath - Unused, retained for API consistency.
9
9
  * @param buffer - Raw DOCX file bytes.
10
10
  */
11
11
  export async function extract(_filePath, buffer) {
12
12
  try {
13
13
  const { extractDocxText } = await import("../chunker/docx.js");
14
- const content = await extractDocxText(buffer);
14
+ const content = extractDocxText(buffer);
15
15
  return { content, ok: true };
16
16
  }
17
17
  catch (err) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "opencode-rag-plugin",
3
- "version": "2.5.0",
3
+ "version": "2.5.1",
4
4
  "description": "OpenCode plugin for local-first RAG-based semantic code search",
5
5
  "type": "module",
6
6
  "main": "./dist/plugin-entry.js",
@@ -112,8 +112,8 @@
112
112
  "apache-arrow": "18.1.0",
113
113
  "chokidar": "^4.0.3",
114
114
  "commander": "^13.1.0",
115
+ "fflate": "^0.8.3",
115
116
  "js-tiktoken": "^1.0.21",
116
- "mammoth": "^1.12.0",
117
117
  "minimatch": "^8.0.7",
118
118
  "p-limit": "^7.3.0",
119
119
  "pdfjs-dist": "^6.3.289",
@@ -149,7 +149,8 @@
149
149
  "uuid": "^11.1.1",
150
150
  "web-tree-sitter": "$web-tree-sitter",
151
151
  "@babel/core": "^7.29.7",
152
- "@opentelemetry/core": "^2.8.0"
152
+ "@opentelemetry/core": "^2.8.0",
153
+ "seroval": "^1.6.8"
153
154
  },
154
155
  "allowScripts": {
155
156
  "opencode-rag-plugin": true,