@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ZIP Archive Extraction Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides functions for extracting files from ZIP archives.
|
|
5
|
+
* Essential for parsing OOXML (DOCX, XLSX, PPTX) and ODF (ODT, ODP, ODS) files,
|
|
6
|
+
* which are all ZIP archives containing XML and media files.
|
|
7
|
+
*
|
|
8
|
+
* Office File Structure:
|
|
9
|
+
* - DOCX: ZIP containing word/document.xml, word/styles.xml, word/media/*, etc.
|
|
10
|
+
* - XLSX: ZIP containing xl/workbook.xml, xl/worksheets/sheet1.xml, etc.
|
|
11
|
+
* - PPTX: ZIP containing ppt/slides/slide1.xml, ppt/media/*, etc.
|
|
12
|
+
* - ODF: Similar structure with content.xml, styles.xml, etc.
|
|
13
|
+
*
|
|
14
|
+
* @module zipUtils
|
|
15
|
+
*/
|
|
16
|
+
import { DecompressionLimits, OfficeParserConfig, SupportedFileType } from '../types.js';
|
|
17
|
+
/**
|
|
18
|
+
* Represents a file extracted from a ZIP archive.
|
|
19
|
+
* Contains the file's path within the archive and its content as a Buffer.
|
|
20
|
+
*/
|
|
21
|
+
export interface ZipFileContent {
|
|
22
|
+
/**
|
|
23
|
+
* The relative path of the file within the ZIP archive.
|
|
24
|
+
* @example "word/document.xml", "xl/worksheets/sheet1.xml", "ppt/slides/slide1.xml"
|
|
25
|
+
*/
|
|
26
|
+
path: string;
|
|
27
|
+
/**
|
|
28
|
+
* The file content as a Node.js Buffer.
|
|
29
|
+
* Can be converted to string for XML files or used directly for binary files (images, etc.).
|
|
30
|
+
* @example Buffer containing XML text or binary image data
|
|
31
|
+
*/
|
|
32
|
+
content: Buffer;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Extracts files from a ZIP archive with optional filtering.
|
|
36
|
+
*
|
|
37
|
+
* This function:
|
|
38
|
+
* 1. Opens the ZIP archive from a Buffer
|
|
39
|
+
* 2. Iterates through all entries in the archive
|
|
40
|
+
* 3. Applies a filter function to determine which files to extract
|
|
41
|
+
* 4. Extracts matching files and returns them as an array
|
|
42
|
+
*
|
|
43
|
+
* Uses lazy entry reading for better memory efficiency with large archives.
|
|
44
|
+
* Files are extracted asynchronously and collected into an array.
|
|
45
|
+
*
|
|
46
|
+
* @param zipInput - The ZIP file as a Node.js Buffer
|
|
47
|
+
* @param filterFn - A predicate function to determine which files to extract.
|
|
48
|
+
* Receives the filename and returns true to extract, false to skip.
|
|
49
|
+
* @param limits - Decompression limits guarding against zip bombs
|
|
50
|
+
* @param config - Parser configuration, so extraction failures honour `onWarning` /
|
|
51
|
+
* `outputErrorToConsole` like every other reported issue
|
|
52
|
+
* @returns A promise resolving to an array of extracted files
|
|
53
|
+
* @throws {Error} If the ZIP file cannot be opened or an entry cannot be read
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```typescript
|
|
57
|
+
* // Extract only XML files from a DOCX
|
|
58
|
+
* const files = await extractFiles(docxBuffer, (fileName) => fileName.endsWith('.xml'));
|
|
59
|
+
*
|
|
60
|
+
* // Extract document.xml specifically
|
|
61
|
+
* const files = await extractFiles(docxBuffer, (fileName) =>
|
|
62
|
+
* fileName === 'word/document.xml'
|
|
63
|
+
* );
|
|
64
|
+
*
|
|
65
|
+
* // Extract all files
|
|
66
|
+
* const allFiles = await extractFiles(zipBuffer, () => true);
|
|
67
|
+
*
|
|
68
|
+
* // Extract everything except media files
|
|
69
|
+
* const files = await extractFiles(zipBuffer, (fileName) =>
|
|
70
|
+
* !fileName.startsWith('word/media/')
|
|
71
|
+
* );
|
|
72
|
+
* ```
|
|
73
|
+
*
|
|
74
|
+
* @see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT ZIP file format specification
|
|
75
|
+
*/
|
|
76
|
+
export declare const extractFiles: (zipInput: Buffer, filterFn: (fileName: string) => boolean, limits: DecompressionLimits, config?: OfficeParserConfig) => Promise<ZipFileContent[]>;
|
|
77
|
+
/**
|
|
78
|
+
* Finds the archive part that every document of a given format must contain, and fails loudly
|
|
79
|
+
* when it is absent.
|
|
80
|
+
*
|
|
81
|
+
* A readable ZIP archive is not by itself a document: an archive can decompress perfectly and
|
|
82
|
+
* still be a renamed photo bundle, a partial upload, or a file mislabeled with the wrong
|
|
83
|
+
* extension. Without this check a parser finds no content to walk and returns an empty AST,
|
|
84
|
+
* which a caller cannot distinguish from a document that genuinely has nothing in it. Every
|
|
85
|
+
* ZIP-backed format has one part it cannot be valid without, so its absence is a hard error.
|
|
86
|
+
*
|
|
87
|
+
* Pass the parser's own regex/predicate for the part rather than a fresh copy of the path, so
|
|
88
|
+
* this check and the code that later reads the part cannot drift apart.
|
|
89
|
+
*
|
|
90
|
+
* @param files - The entries extracted from the archive
|
|
91
|
+
* @param matcher - Predicate identifying the required part by its path within the archive
|
|
92
|
+
* @param config - Parser configuration, so the error is reported through the caller's handlers
|
|
93
|
+
* @param info - The document format and the human-readable part name, used in the message
|
|
94
|
+
* @returns The matching entry
|
|
95
|
+
* @throws {Error} A typed REQUIRED_PART_MISSING error when no entry matches
|
|
96
|
+
*
|
|
97
|
+
* @example
|
|
98
|
+
* ```typescript
|
|
99
|
+
* const document = findRequiredPart(files, p => !!p.match(documentFileRegex), config,
|
|
100
|
+
* { fileType: 'docx', part: 'word/document.xml' });
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
export declare const findRequiredPart: (files: ZipFileContent[], matcher: (path: string) => boolean, config: OfficeParserConfig, info: {
|
|
104
|
+
fileType: string;
|
|
105
|
+
part: string;
|
|
106
|
+
}) => ZipFileContent;
|
|
107
|
+
/**
|
|
108
|
+
* Resolves which office format a ZIP archive actually holds, by reading the part that names it.
|
|
109
|
+
*
|
|
110
|
+
* This exists because magic-byte sniffing is a heuristic that gives up. `file-type` identifies an
|
|
111
|
+
* OOXML package by parsing `[Content_Types].xml`, but it walks the archive under fixed budgets:
|
|
112
|
+
* at most 1024 entries, and (for entries whose sizes are deferred to a trailing data descriptor,
|
|
113
|
+
* general-purpose flag bit 3) about 1 MiB of scanning to locate those descriptors. An archive
|
|
114
|
+
* that puts enough data before `[Content_Types].xml` to exhaust either budget is reported as a
|
|
115
|
+
* generic `zip`, which previously surfaced to the caller as "add support for zip files" for a
|
|
116
|
+
* perfectly valid document. Both layouts occur in the wild: streaming ZIP writers set bit 3, and
|
|
117
|
+
* a media-heavy deck can hold more than 1024 parts.
|
|
118
|
+
*
|
|
119
|
+
* We already ship a ZIP reader that has neither limitation, so rather than guessing from the
|
|
120
|
+
* first bytes this opens the archive and reads the declaration directly. It is deliberately the
|
|
121
|
+
* fallback rather than the primary check, since the byte-level sniff is far cheaper and settles
|
|
122
|
+
* every non-ZIP format.
|
|
123
|
+
*
|
|
124
|
+
* @param zipInput - The candidate archive
|
|
125
|
+
* @param limits - Decompression limits, so sniffing an untrusted file stays bounded
|
|
126
|
+
* @returns The format the archive declares, or `undefined` if it declares none or cannot be read
|
|
127
|
+
*
|
|
128
|
+
* @example
|
|
129
|
+
* ```typescript
|
|
130
|
+
* // A presentation whose [Content_Types].xml sits behind 2 MiB of streamed entries
|
|
131
|
+
* await detectOfficeTypeFromZip(buffer, limits); // -> 'pptx'
|
|
132
|
+
* ```
|
|
133
|
+
*/
|
|
134
|
+
export declare const detectOfficeTypeFromZip: (zipInput: Buffer, limits: DecompressionLimits) => Promise<SupportedFileType | undefined>;
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ZIP Archive Extraction Utilities
|
|
4
|
+
*
|
|
5
|
+
* Provides functions for extracting files from ZIP archives.
|
|
6
|
+
* Essential for parsing OOXML (DOCX, XLSX, PPTX) and ODF (ODT, ODP, ODS) files,
|
|
7
|
+
* which are all ZIP archives containing XML and media files.
|
|
8
|
+
*
|
|
9
|
+
* Office File Structure:
|
|
10
|
+
* - DOCX: ZIP containing word/document.xml, word/styles.xml, word/media/*, etc.
|
|
11
|
+
* - XLSX: ZIP containing xl/workbook.xml, xl/worksheets/sheet1.xml, etc.
|
|
12
|
+
* - PPTX: ZIP containing ppt/slides/slide1.xml, ppt/media/*, etc.
|
|
13
|
+
* - ODF: Similar structure with content.xml, styles.xml, etc.
|
|
14
|
+
*
|
|
15
|
+
* @module zipUtils
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.detectOfficeTypeFromZip = exports.findRequiredPart = exports.extractFiles = void 0;
|
|
19
|
+
const fflate_1 = require("fflate");
|
|
20
|
+
const types_js_1 = require("../types.js");
|
|
21
|
+
const errorUtils_js_1 = require("./errorUtils.js");
|
|
22
|
+
/**
|
|
23
|
+
* Signature of the End Of Central Directory record ("PK\x05\x06"), the trailer every ZIP
|
|
24
|
+
* archive ends with. Its presence is what distinguishes a complete archive from one that
|
|
25
|
+
* was cut off in transfer.
|
|
26
|
+
*/
|
|
27
|
+
const EOCD_SIGNATURE = Buffer.from([0x50, 0x4b, 0x05, 0x06]);
|
|
28
|
+
/** Size of the fixed portion of an End Of Central Directory record, in bytes. */
|
|
29
|
+
const EOCD_RECORD_MIN_BYTES = 22;
|
|
30
|
+
/** Maximum size of the optional archive comment that may trail the EOCD record, in bytes. */
|
|
31
|
+
const ZIP_MAX_COMMENT_BYTES = 65535;
|
|
32
|
+
/**
|
|
33
|
+
* How far back from the end of the input the EOCD record may start. The record is last in
|
|
34
|
+
* the file apart from its own variable-length comment, so searching this window is
|
|
35
|
+
* sufficient and bounded regardless of archive size.
|
|
36
|
+
*/
|
|
37
|
+
const EOCD_SEARCH_WINDOW_BYTES = EOCD_RECORD_MIN_BYTES + ZIP_MAX_COMMENT_BYTES;
|
|
38
|
+
/**
|
|
39
|
+
* Extracts files from a ZIP archive with optional filtering.
|
|
40
|
+
*
|
|
41
|
+
* This function:
|
|
42
|
+
* 1. Opens the ZIP archive from a Buffer
|
|
43
|
+
* 2. Iterates through all entries in the archive
|
|
44
|
+
* 3. Applies a filter function to determine which files to extract
|
|
45
|
+
* 4. Extracts matching files and returns them as an array
|
|
46
|
+
*
|
|
47
|
+
* Uses lazy entry reading for better memory efficiency with large archives.
|
|
48
|
+
* Files are extracted asynchronously and collected into an array.
|
|
49
|
+
*
|
|
50
|
+
* @param zipInput - The ZIP file as a Node.js Buffer
|
|
51
|
+
* @param filterFn - A predicate function to determine which files to extract.
|
|
52
|
+
* Receives the filename and returns true to extract, false to skip.
|
|
53
|
+
* @param limits - Decompression limits guarding against zip bombs
|
|
54
|
+
* @param config - Parser configuration, so extraction failures honour `onWarning` /
|
|
55
|
+
* `outputErrorToConsole` like every other reported issue
|
|
56
|
+
* @returns A promise resolving to an array of extracted files
|
|
57
|
+
* @throws {Error} If the ZIP file cannot be opened or an entry cannot be read
|
|
58
|
+
*
|
|
59
|
+
* @example
|
|
60
|
+
* ```typescript
|
|
61
|
+
* // Extract only XML files from a DOCX
|
|
62
|
+
* const files = await extractFiles(docxBuffer, (fileName) => fileName.endsWith('.xml'));
|
|
63
|
+
*
|
|
64
|
+
* // Extract document.xml specifically
|
|
65
|
+
* const files = await extractFiles(docxBuffer, (fileName) =>
|
|
66
|
+
* fileName === 'word/document.xml'
|
|
67
|
+
* );
|
|
68
|
+
*
|
|
69
|
+
* // Extract all files
|
|
70
|
+
* const allFiles = await extractFiles(zipBuffer, () => true);
|
|
71
|
+
*
|
|
72
|
+
* // Extract everything except media files
|
|
73
|
+
* const files = await extractFiles(zipBuffer, (fileName) =>
|
|
74
|
+
* !fileName.startsWith('word/media/')
|
|
75
|
+
* );
|
|
76
|
+
* ```
|
|
77
|
+
*
|
|
78
|
+
* @see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT ZIP file format specification
|
|
79
|
+
*/
|
|
80
|
+
const extractFiles = (zipInput, filterFn, limits, config) => {
|
|
81
|
+
const maxUncompressedBytes = limits?.maxUncompressedBytes !== undefined && Number.isFinite(limits.maxUncompressedBytes) && limits.maxUncompressedBytes >= 0
|
|
82
|
+
? limits.maxUncompressedBytes
|
|
83
|
+
: 512 * 1024 * 1024;
|
|
84
|
+
const maxZipEntries = limits?.maxZipEntries !== undefined && Number.isFinite(limits.maxZipEntries) && limits.maxZipEntries >= 0
|
|
85
|
+
? limits.maxZipEntries
|
|
86
|
+
: 10000;
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
// Decompress as a stream and cap on the ACTUAL inflated byte count rather than
|
|
89
|
+
// the size declared in the ZIP header. The declared size is attacker-controlled,
|
|
90
|
+
// so a "zip bomb" can understate it and still inflate to gigabytes; counting real
|
|
91
|
+
// output bytes and aborting once the limit is crossed is the only reliable guard.
|
|
92
|
+
const results = [];
|
|
93
|
+
let totalEntryCount = 0;
|
|
94
|
+
let actualTotalBytes = 0;
|
|
95
|
+
let pendingFiles = 0;
|
|
96
|
+
let pushComplete = false;
|
|
97
|
+
let settled = false;
|
|
98
|
+
const fail = (err) => {
|
|
99
|
+
if (settled)
|
|
100
|
+
return;
|
|
101
|
+
settled = true;
|
|
102
|
+
reject(err);
|
|
103
|
+
};
|
|
104
|
+
const maybeResolve = () => {
|
|
105
|
+
if (!settled && pushComplete && pendingFiles === 0) {
|
|
106
|
+
settled = true;
|
|
107
|
+
resolve(results);
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
const unzipper = new fflate_1.Unzip((file) => {
|
|
111
|
+
if (settled)
|
|
112
|
+
return;
|
|
113
|
+
totalEntryCount++;
|
|
114
|
+
if (totalEntryCount > maxZipEntries) {
|
|
115
|
+
fail((0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.ZIP_ENTRY_COUNT_LIMIT_EXCEEDED, config, maxZipEntries));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (!filterFn(file.name))
|
|
119
|
+
return;
|
|
120
|
+
const name = file.name;
|
|
121
|
+
const chunks = [];
|
|
122
|
+
pendingFiles++;
|
|
123
|
+
file.ondata = (err, chunk, final) => {
|
|
124
|
+
if (settled)
|
|
125
|
+
return;
|
|
126
|
+
if (err) {
|
|
127
|
+
fail(err);
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
if (chunk && chunk.length) {
|
|
131
|
+
actualTotalBytes += chunk.length;
|
|
132
|
+
if (actualTotalBytes > maxUncompressedBytes) {
|
|
133
|
+
fail((0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.ZIP_SIZE_LIMIT_EXCEEDED, config, maxUncompressedBytes));
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
chunks.push(Buffer.from(chunk));
|
|
137
|
+
}
|
|
138
|
+
if (final) {
|
|
139
|
+
results.push({ path: name, content: Buffer.concat(chunks) });
|
|
140
|
+
pendingFiles--;
|
|
141
|
+
maybeResolve();
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
try {
|
|
145
|
+
file.start();
|
|
146
|
+
}
|
|
147
|
+
catch (e) {
|
|
148
|
+
fail(e);
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
unzipper.register(fflate_1.UnzipInflate);
|
|
152
|
+
const src = new Uint8Array(zipInput.buffer, zipInput.byteOffset, zipInput.byteLength);
|
|
153
|
+
// Feed the compressed input in bounded chunks so the actual-size check can abort a
|
|
154
|
+
// bomb early; the worst-case overshoot is one chunk's worth of inflation.
|
|
155
|
+
const PUSH_CHUNK = 1 << 16; // 64 KiB
|
|
156
|
+
try {
|
|
157
|
+
let offset = 0;
|
|
158
|
+
while (offset < src.length && !settled) {
|
|
159
|
+
const end = Math.min(offset + PUSH_CHUNK, src.length);
|
|
160
|
+
unzipper.push(src.subarray(offset, end), end >= src.length);
|
|
161
|
+
offset = end;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
catch (e) {
|
|
165
|
+
fail(e);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
pushComplete = true;
|
|
169
|
+
// fflate's streaming Unzip emits nothing, and no error, when the input is not a
|
|
170
|
+
// ZIP archive, unlike the central-directory-based unzip() this replaced (which
|
|
171
|
+
// rejected with "invalid zip data"). Zero entries can never be a valid document
|
|
172
|
+
// here, since every ZIP-backed format requires at least one part, so treat it as
|
|
173
|
+
// corrupt input rather than resolving into an empty, successfully-parsed document.
|
|
174
|
+
//
|
|
175
|
+
// This is checked before the truncation check below because it produces the better
|
|
176
|
+
// message for input that is not an archive at all, and because garbage that happens
|
|
177
|
+
// to contain the EOCD signature would otherwise slip past.
|
|
178
|
+
if (totalEntryCount === 0) {
|
|
179
|
+
fail((0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.ZIP_NO_ENTRIES_FOUND, config));
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
// The streaming reader recovers entries from local file headers alone, so an archive
|
|
183
|
+
// cut short still yields whatever entries preceded the cut - silently, and possibly
|
|
184
|
+
// missing parts that came after it. The central-directory-based reader used before
|
|
185
|
+
// 7.3.0 rejected such input outright. Requiring the trailer that terminates every
|
|
186
|
+
// complete archive restores that: absent it, the data is truncated and the entries
|
|
187
|
+
// recovered cannot be trusted to be the whole document.
|
|
188
|
+
//
|
|
189
|
+
// Deliberately not gated on pendingFiles: when a cut lands inside an entry's
|
|
190
|
+
// compressed data that entry's final callback never fires, so this is also what
|
|
191
|
+
// settles the promise instead of leaving the caller waiting forever.
|
|
192
|
+
const tail = zipInput.subarray(Math.max(0, zipInput.length - EOCD_SEARCH_WINDOW_BYTES));
|
|
193
|
+
const eocdIndex = tail.lastIndexOf(EOCD_SIGNATURE);
|
|
194
|
+
if (eocdIndex === -1 || eocdIndex + EOCD_RECORD_MIN_BYTES > tail.length) {
|
|
195
|
+
fail((0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.ZIP_TRUNCATED, config));
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
maybeResolve();
|
|
199
|
+
});
|
|
200
|
+
};
|
|
201
|
+
exports.extractFiles = extractFiles;
|
|
202
|
+
/**
|
|
203
|
+
* Finds the archive part that every document of a given format must contain, and fails loudly
|
|
204
|
+
* when it is absent.
|
|
205
|
+
*
|
|
206
|
+
* A readable ZIP archive is not by itself a document: an archive can decompress perfectly and
|
|
207
|
+
* still be a renamed photo bundle, a partial upload, or a file mislabeled with the wrong
|
|
208
|
+
* extension. Without this check a parser finds no content to walk and returns an empty AST,
|
|
209
|
+
* which a caller cannot distinguish from a document that genuinely has nothing in it. Every
|
|
210
|
+
* ZIP-backed format has one part it cannot be valid without, so its absence is a hard error.
|
|
211
|
+
*
|
|
212
|
+
* Pass the parser's own regex/predicate for the part rather than a fresh copy of the path, so
|
|
213
|
+
* this check and the code that later reads the part cannot drift apart.
|
|
214
|
+
*
|
|
215
|
+
* @param files - The entries extracted from the archive
|
|
216
|
+
* @param matcher - Predicate identifying the required part by its path within the archive
|
|
217
|
+
* @param config - Parser configuration, so the error is reported through the caller's handlers
|
|
218
|
+
* @param info - The document format and the human-readable part name, used in the message
|
|
219
|
+
* @returns The matching entry
|
|
220
|
+
* @throws {Error} A typed REQUIRED_PART_MISSING error when no entry matches
|
|
221
|
+
*
|
|
222
|
+
* @example
|
|
223
|
+
* ```typescript
|
|
224
|
+
* const document = findRequiredPart(files, p => !!p.match(documentFileRegex), config,
|
|
225
|
+
* { fileType: 'docx', part: 'word/document.xml' });
|
|
226
|
+
* ```
|
|
227
|
+
*/
|
|
228
|
+
const findRequiredPart = (files, matcher, config, info) => {
|
|
229
|
+
const found = files.find(file => matcher(file.path));
|
|
230
|
+
if (!found)
|
|
231
|
+
throw (0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.REQUIRED_PART_MISSING, config, info);
|
|
232
|
+
return found;
|
|
233
|
+
};
|
|
234
|
+
exports.findRequiredPart = findRequiredPart;
|
|
235
|
+
/** The part naming an OOXML package's document type. */
|
|
236
|
+
const OOXML_CONTENT_TYPES_PATH = '[Content_Types].xml';
|
|
237
|
+
/** The part naming an ODF or EPUB package's document type. */
|
|
238
|
+
const ODF_MIMETYPE_PATH = 'mimetype';
|
|
239
|
+
/**
|
|
240
|
+
* Substrings of the main-part content type each OOXML format declares in
|
|
241
|
+
* `[Content_Types].xml`, matched as plain text because only this one value is needed.
|
|
242
|
+
*/
|
|
243
|
+
const OOXML_MAIN_CONTENT_TYPES = [
|
|
244
|
+
['wordprocessingml.document.main+xml', 'docx'],
|
|
245
|
+
['spreadsheetml.sheet.main+xml', 'xlsx'],
|
|
246
|
+
['presentationml.presentation.main+xml', 'pptx'],
|
|
247
|
+
];
|
|
248
|
+
/** Exact `mimetype` entry contents for the packages that carry one. */
|
|
249
|
+
const PACKAGE_MIMETYPES = {
|
|
250
|
+
'application/vnd.oasis.opendocument.text': 'odt',
|
|
251
|
+
'application/vnd.oasis.opendocument.spreadsheet': 'ods',
|
|
252
|
+
'application/vnd.oasis.opendocument.presentation': 'odp',
|
|
253
|
+
'application/epub+zip': 'epub',
|
|
254
|
+
};
|
|
255
|
+
/** First two bytes of every ZIP local file header ("PK"). */
|
|
256
|
+
const ZIP_MAGIC_BYTES = [0x50, 0x4b];
|
|
257
|
+
/**
|
|
258
|
+
* Reporting is suppressed while sniffing: the input is not yet known to be a document, so a
|
|
259
|
+
* failure here is an inconclusive guess rather than something the caller did wrong. Without a
|
|
260
|
+
* config, `getOfficeError` would write these to the console.
|
|
261
|
+
*/
|
|
262
|
+
const SILENT_DETECTION_CONFIG = { outputErrorToConsole: false };
|
|
263
|
+
/** Whether a buffer starts with the ZIP local file header signature. */
|
|
264
|
+
const looksLikeZip = (buffer) => buffer.length >= ZIP_MAGIC_BYTES.length && ZIP_MAGIC_BYTES.every((byte, i) => buffer[i] === byte);
|
|
265
|
+
/**
|
|
266
|
+
* How much a type sniff may inflate before giving up, in bytes.
|
|
267
|
+
*
|
|
268
|
+
* The two parts read here name the format and nothing else, so they are tiny in any real
|
|
269
|
+
* document: a few hundred bytes of `mimetype`, a few kilobytes of `[Content_Types].xml`. A
|
|
270
|
+
* crafted archive could declare them as hundreds of megabytes, and since the document is
|
|
271
|
+
* inflated again during the parse that follows, honouring the full decompression budget here
|
|
272
|
+
* would let a single call spend it twice. This cap keeps sniffing cheap; an archive that
|
|
273
|
+
* exceeds it is simply reported as unidentified.
|
|
274
|
+
*/
|
|
275
|
+
const MAX_DETECTION_INFLATED_BYTES = 4 * 1024 * 1024;
|
|
276
|
+
/**
|
|
277
|
+
* Resolves which office format a ZIP archive actually holds, by reading the part that names it.
|
|
278
|
+
*
|
|
279
|
+
* This exists because magic-byte sniffing is a heuristic that gives up. `file-type` identifies an
|
|
280
|
+
* OOXML package by parsing `[Content_Types].xml`, but it walks the archive under fixed budgets:
|
|
281
|
+
* at most 1024 entries, and (for entries whose sizes are deferred to a trailing data descriptor,
|
|
282
|
+
* general-purpose flag bit 3) about 1 MiB of scanning to locate those descriptors. An archive
|
|
283
|
+
* that puts enough data before `[Content_Types].xml` to exhaust either budget is reported as a
|
|
284
|
+
* generic `zip`, which previously surfaced to the caller as "add support for zip files" for a
|
|
285
|
+
* perfectly valid document. Both layouts occur in the wild: streaming ZIP writers set bit 3, and
|
|
286
|
+
* a media-heavy deck can hold more than 1024 parts.
|
|
287
|
+
*
|
|
288
|
+
* We already ship a ZIP reader that has neither limitation, so rather than guessing from the
|
|
289
|
+
* first bytes this opens the archive and reads the declaration directly. It is deliberately the
|
|
290
|
+
* fallback rather than the primary check, since the byte-level sniff is far cheaper and settles
|
|
291
|
+
* every non-ZIP format.
|
|
292
|
+
*
|
|
293
|
+
* @param zipInput - The candidate archive
|
|
294
|
+
* @param limits - Decompression limits, so sniffing an untrusted file stays bounded
|
|
295
|
+
* @returns The format the archive declares, or `undefined` if it declares none or cannot be read
|
|
296
|
+
*
|
|
297
|
+
* @example
|
|
298
|
+
* ```typescript
|
|
299
|
+
* // A presentation whose [Content_Types].xml sits behind 2 MiB of streamed entries
|
|
300
|
+
* await detectOfficeTypeFromZip(buffer, limits); // -> 'pptx'
|
|
301
|
+
* ```
|
|
302
|
+
*/
|
|
303
|
+
const detectOfficeTypeFromZip = async (zipInput, limits) => {
|
|
304
|
+
if (!looksLikeZip(zipInput))
|
|
305
|
+
return undefined;
|
|
306
|
+
let files;
|
|
307
|
+
try {
|
|
308
|
+
files = await (0, exports.extractFiles)(zipInput, name => name === OOXML_CONTENT_TYPES_PATH || name === ODF_MIMETYPE_PATH,
|
|
309
|
+
// Never inflate more for a sniff than the caller already allows for the parse, and
|
|
310
|
+
// never more than a sniff could legitimately need.
|
|
311
|
+
{
|
|
312
|
+
...limits,
|
|
313
|
+
maxUncompressedBytes: Math.min(limits?.maxUncompressedBytes ?? MAX_DETECTION_INFLATED_BYTES, MAX_DETECTION_INFLATED_BYTES),
|
|
314
|
+
}, SILENT_DETECTION_CONFIG);
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
// Unreadable, truncated, or not an archive at all. The caller keeps whatever the
|
|
318
|
+
// byte-level sniff decided, and the parser it dispatches to reports the real problem.
|
|
319
|
+
return undefined;
|
|
320
|
+
}
|
|
321
|
+
// ODF and EPUB state their type outright, so prefer that over inspecting OOXML parts.
|
|
322
|
+
const mimetypeEntry = files.find(file => file.path === ODF_MIMETYPE_PATH);
|
|
323
|
+
if (mimetypeEntry) {
|
|
324
|
+
const declared = PACKAGE_MIMETYPES[mimetypeEntry.content.toString().trim()];
|
|
325
|
+
if (declared)
|
|
326
|
+
return declared;
|
|
327
|
+
}
|
|
328
|
+
const contentTypesEntry = files.find(file => file.path === OOXML_CONTENT_TYPES_PATH);
|
|
329
|
+
if (contentTypesEntry) {
|
|
330
|
+
const contentTypes = contentTypesEntry.content.toString();
|
|
331
|
+
const match = OOXML_MAIN_CONTENT_TYPES.find(([marker]) => contentTypes.includes(marker));
|
|
332
|
+
if (match)
|
|
333
|
+
return match[1];
|
|
334
|
+
}
|
|
335
|
+
return undefined;
|
|
336
|
+
};
|
|
337
|
+
exports.detectOfficeTypeFromZip = detectOfficeTypeFromZip;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "officeparser",
|
|
3
|
+
"version": "7.8.0",
|
|
4
|
+
"description": "A robust, strictly-typed Node.js and Browser library for parsing office files (.docx, .pptx, .xlsx, .odt, .odp, .ods, .pdf, .rtf, .csv, .md, .html, .epub) and generating high-fidelity outputs in Markdown, HTML, CSV, RTF, PDF, EPUB, and RAG-focused chunks.",
|
|
5
|
+
"funding": "https://github.com/sponsors/harshankur",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"module": "dist/index.mjs",
|
|
8
|
+
"types": "dist/index.d.ts",
|
|
9
|
+
"browser": "./dist/officeparser.browser.mjs",
|
|
10
|
+
"exports": {
|
|
11
|
+
".": {
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"browser": "./dist/officeparser.browser.mjs",
|
|
14
|
+
"import": "./dist/index.mjs",
|
|
15
|
+
"require": "./dist/index.js"
|
|
16
|
+
},
|
|
17
|
+
"./slim": {
|
|
18
|
+
"types": "./dist/officeparser.browser.slim.d.ts",
|
|
19
|
+
"browser": "./dist/officeparser.browser.slim.mjs",
|
|
20
|
+
"import": "./dist/officeparser.browser.slim.mjs"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"sideEffects": false,
|
|
24
|
+
"engines": {
|
|
25
|
+
"node": ">=18.0.0"
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"dist"
|
|
29
|
+
],
|
|
30
|
+
"scripts": {
|
|
31
|
+
"build": "npm run sync:versions && npm run build:node && npm run build:esm-wrapper && npm run build:browser:types && npm run build:browser",
|
|
32
|
+
"build:node": "tsc",
|
|
33
|
+
"build:esm-wrapper": "node scripts/generate-esm-wrapper.js",
|
|
34
|
+
"build:browser:types": "dts-bundle-generator --no-check -o dist/officeparser.browser.d.ts src/index.ts && node -e \"require('fs').copyFileSync('dist/officeparser.browser.d.ts','dist/officeparser.browser.slim.d.ts')\"",
|
|
35
|
+
"build:browser": "node build_browser.js && npm run sync:docs",
|
|
36
|
+
"sync:versions": "node scripts/sync-pdfjs-versions.js",
|
|
37
|
+
"sync:docs": "node -e \"const fs=require('fs');fs.mkdirSync('docs/dist',{recursive:true});for(const f of ['officeparser.browser.iife.js','officeparser.browser.mjs','officeparser.browser.slim.iife.js','officeparser.browser.slim.mjs'])fs.copyFileSync('dist/'+f,'docs/dist/'+f);fs.mkdirSync('docs/test/files',{recursive:true});fs.cpSync('test/files','docs/test/files',{recursive:true})\"",
|
|
38
|
+
"lint": "eslint src",
|
|
39
|
+
"test": "npm run lint && npm run test:clean && npm run build && npm run test:license && npm run test:artifacts && npm run test:parser && npm run test:exhaustive && npm run test:generator && npm run test:security && npm run test:cli",
|
|
40
|
+
"test:fast": "npm run lint && npm run test:parser:fast && npm run test:exhaustive && npm run test:generator:fast && npm run test:security && npm run test:cli:fast",
|
|
41
|
+
"test:exhaustive": "npx tsx test/testExhaustive.ts",
|
|
42
|
+
"test:security": "npx tsx test/security/testSanitization.ts",
|
|
43
|
+
"test:baseline": "npm run test:parser:baseline && npm run test:generator:baseline",
|
|
44
|
+
"test:parser": "npx tsx test/parser/testOfficeParser.ts",
|
|
45
|
+
"test:parser:fast": "npx tsx test/parser/testOfficeParser.ts fast",
|
|
46
|
+
"test:parser:baseline": "npx tsx test/parser/testOfficeParser.ts baseline",
|
|
47
|
+
"test:generator": "npx tsx test/generator/testOfficeGenerator.ts",
|
|
48
|
+
"test:generator:fast": "npx tsx test/generator/testOfficeGenerator.ts fast",
|
|
49
|
+
"test:generator:baseline": "npx tsx test/generator/testOfficeGenerator.ts baseline",
|
|
50
|
+
"test:artifacts": "npx tsx test/testShippingArtifacts.ts",
|
|
51
|
+
"test:cli": "npx tsx test/cli/testCli.ts",
|
|
52
|
+
"test:cli:fast": "npx tsx test/cli/testCli.ts fast",
|
|
53
|
+
"test:visualizer": "node test/testVisualizer.js",
|
|
54
|
+
"test:integration": "node test/testIntegration.js",
|
|
55
|
+
"test:license": "npm run sbom && node scripts/validate-licenses.js",
|
|
56
|
+
"test:clean": "node -e \"for(const d of ['test/results','test/generator/results','test/generator/output','test/parser/results','test/parser/output','test/cli/results'])require('fs').rmSync(d,{recursive:true,force:true})\"",
|
|
57
|
+
"clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && npm run test:clean",
|
|
58
|
+
"sbom": "npx --yes @cyclonedx/cyclonedx-npm --output-format json --output-file dist/sbom.cdx.json --omit dev",
|
|
59
|
+
"prepublishOnly": "npm run build",
|
|
60
|
+
"prepare": "husky"
|
|
61
|
+
},
|
|
62
|
+
"repository": {
|
|
63
|
+
"type": "git",
|
|
64
|
+
"url": "git+https://github.com/harshankur/officeParser.git"
|
|
65
|
+
},
|
|
66
|
+
"bin": {
|
|
67
|
+
"officeparser": "dist/cli.js"
|
|
68
|
+
},
|
|
69
|
+
"publishConfig": {
|
|
70
|
+
"access": "public",
|
|
71
|
+
"provenance": true
|
|
72
|
+
},
|
|
73
|
+
"keywords": [
|
|
74
|
+
"office",
|
|
75
|
+
"docx",
|
|
76
|
+
"pptx",
|
|
77
|
+
"xlsx",
|
|
78
|
+
"odt",
|
|
79
|
+
"odp",
|
|
80
|
+
"ods",
|
|
81
|
+
"pdf",
|
|
82
|
+
"rtf",
|
|
83
|
+
"csv",
|
|
84
|
+
"parser",
|
|
85
|
+
"text extraction",
|
|
86
|
+
"document parser",
|
|
87
|
+
"word",
|
|
88
|
+
"excel",
|
|
89
|
+
"powerpoint",
|
|
90
|
+
"spreadsheet",
|
|
91
|
+
"presentation",
|
|
92
|
+
"slides",
|
|
93
|
+
"ast",
|
|
94
|
+
"ocr",
|
|
95
|
+
"typescript",
|
|
96
|
+
"browser",
|
|
97
|
+
"metadata",
|
|
98
|
+
"formatting",
|
|
99
|
+
"attachments",
|
|
100
|
+
"tesseract",
|
|
101
|
+
"pdf.js",
|
|
102
|
+
"structured-data",
|
|
103
|
+
"openoffice",
|
|
104
|
+
"libreoffice",
|
|
105
|
+
"generator",
|
|
106
|
+
"rag",
|
|
107
|
+
"chunking",
|
|
108
|
+
"markdown",
|
|
109
|
+
"html",
|
|
110
|
+
"epub",
|
|
111
|
+
"ebook"
|
|
112
|
+
],
|
|
113
|
+
"author": "Harsh Ankur",
|
|
114
|
+
"license": "MIT",
|
|
115
|
+
"bugs": {
|
|
116
|
+
"url": "https://github.com/harshankur/officeParser/issues"
|
|
117
|
+
},
|
|
118
|
+
"homepage": "https://officeparser.harshankur.com",
|
|
119
|
+
"dependencies": {
|
|
120
|
+
"@xmldom/xmldom": "^0.9.10",
|
|
121
|
+
"fflate": "^0.8.3",
|
|
122
|
+
"file-type": "^22.0.1",
|
|
123
|
+
"pdfjs-dist": "6.1.200",
|
|
124
|
+
"tesseract.js": "^7.0.0"
|
|
125
|
+
},
|
|
126
|
+
"peerDependenciesMeta": {
|
|
127
|
+
"puppeteer": {
|
|
128
|
+
"optional": true
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
"devDependencies": {
|
|
132
|
+
"@types/node": "^26.1.1",
|
|
133
|
+
"@typescript-eslint/eslint-plugin": "^8.65.0",
|
|
134
|
+
"@typescript-eslint/parser": "^8.65.0",
|
|
135
|
+
"buffer": "^6.0.3",
|
|
136
|
+
"dts-bundle-generator": "^9.5.1",
|
|
137
|
+
"esbuild": "^0.28.1",
|
|
138
|
+
"esbuild-plugins-node-modules-polyfill": "^1.8.2",
|
|
139
|
+
"eslint": "^10.8.0",
|
|
140
|
+
"husky": "^9.1.7",
|
|
141
|
+
"postject": "^1.0.0-alpha.6",
|
|
142
|
+
"process": "^0.11.10",
|
|
143
|
+
"puppeteer": "^24.43.1",
|
|
144
|
+
"tsx": "^4.23.1",
|
|
145
|
+
"typescript": "^6.0.3"
|
|
146
|
+
}
|
|
147
|
+
}
|