@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* officeparser CLI
|
|
5
|
+
*
|
|
6
|
+
* Allows running officeparser from the command line:
|
|
7
|
+
* npx officeparser file.docx
|
|
8
|
+
* officeparser file.docx --to=text
|
|
9
|
+
* officeparser file.docx --ocr --extractAttachments
|
|
10
|
+
*
|
|
11
|
+
* Options (--key=value, --key value, or bare flags):
|
|
12
|
+
* --to=json|text|md|html|csv|rtf|pdf|epub|chunks Convert AST to specified format (default: json)
|
|
13
|
+
* --output=path Save result to a file
|
|
14
|
+
* --fileType=docx|xlsx|... Override file type detection
|
|
15
|
+
* --ocr Enable OCR for images (default: false)
|
|
16
|
+
* --ocrConfig.language=eng OCR language (default: eng)
|
|
17
|
+
* --extractAttachments Extract embedded attachments (default: false)
|
|
18
|
+
* --ignoreNotes Ignore footnotes/endnotes/speaker notes (default: false)
|
|
19
|
+
* --ignoreComments Ignore inline comments (default: false)
|
|
20
|
+
* --ignoreHeadersAndFooters Ignore headers and footers (default: false)
|
|
21
|
+
* --ignoreSlideMasters Ignore slide masters (default: false)
|
|
22
|
+
* --ignoreInternalLinks Ignore internal links (default: false)
|
|
23
|
+
* --includeRawContent Include raw content in AST (default: false)
|
|
24
|
+
* --serializeRawContent Include stringified XML in metadata (default: true)
|
|
25
|
+
* --preserveXmlWhitespace Keep raw formatting space (default: false)
|
|
26
|
+
* --includeBreakNodes Include break nodes (DOCX only, default: false)
|
|
27
|
+
* --verbose Show full error stack traces and warning logs
|
|
28
|
+
*/
|
|
29
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
30
|
+
if (k2 === undefined) k2 = k;
|
|
31
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
32
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
33
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
34
|
+
}
|
|
35
|
+
Object.defineProperty(o, k2, desc);
|
|
36
|
+
}) : (function(o, m, k, k2) {
|
|
37
|
+
if (k2 === undefined) k2 = k;
|
|
38
|
+
o[k2] = m[k];
|
|
39
|
+
}));
|
|
40
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
41
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
42
|
+
}) : function(o, v) {
|
|
43
|
+
o["default"] = v;
|
|
44
|
+
});
|
|
45
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
46
|
+
var ownKeys = function(o) {
|
|
47
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
48
|
+
var ar = [];
|
|
49
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
50
|
+
return ar;
|
|
51
|
+
};
|
|
52
|
+
return ownKeys(o);
|
|
53
|
+
};
|
|
54
|
+
return function (mod) {
|
|
55
|
+
if (mod && mod.__esModule) return mod;
|
|
56
|
+
var result = {};
|
|
57
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
58
|
+
__setModuleDefault(result, mod);
|
|
59
|
+
return result;
|
|
60
|
+
};
|
|
61
|
+
})();
|
|
62
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
63
|
+
const OfficeParser_js_1 = require("./OfficeParser.js");
|
|
64
|
+
const OfficeGenerator_js_1 = require("./OfficeGenerator.js");
|
|
65
|
+
const fs = __importStar(require("fs"));
|
|
66
|
+
const args = process.argv.slice(2);
|
|
67
|
+
let fileArg;
|
|
68
|
+
let showHelp = false;
|
|
69
|
+
let toFlagOption;
|
|
70
|
+
let formatFlagOption;
|
|
71
|
+
let toTextOption;
|
|
72
|
+
let verbose = false;
|
|
73
|
+
let outputFile;
|
|
74
|
+
// Parser and Generator configuration objects that will be populated by command line options.
|
|
75
|
+
const config = {};
|
|
76
|
+
const generatorConfig = {};
|
|
77
|
+
// Known boolean configurations to validate user input against.
|
|
78
|
+
const knownParserBooleans = new Set([
|
|
79
|
+
'ocr', 'extractAttachments', 'ignoreNotes', 'ignoreComments',
|
|
80
|
+
'ignoreHeadersAndFooters', 'ignoreSlideMasters', 'ignoreInternalLinks',
|
|
81
|
+
'includeRawContent', 'serializeRawContent', 'preserveXmlWhitespace', 'includeBreakNodes'
|
|
82
|
+
]);
|
|
83
|
+
const knownGeneratorBooleans = new Set([
|
|
84
|
+
'includeFormatting', 'generateIds', 'renderMetadata', 'includeImages', 'includeCharts', 'ignoreInternalLinks'
|
|
85
|
+
]);
|
|
86
|
+
// Prefixes used to identify configurations targeted for the generator instead of the parser.
|
|
87
|
+
const generatorPrefixes = [
|
|
88
|
+
'generatorConfig.', 'htmlConfig.', 'csvConfig.', 'textConfig.', 'mdConfig.', 'pdfConfig.', 'rtfConfig.', 'chunksConfig.'
|
|
89
|
+
];
|
|
90
|
+
// Trackers to detect if deprecated/legacy options were used to log helpful warnings.
|
|
91
|
+
let usedFormat = false;
|
|
92
|
+
let usedToText = false;
|
|
93
|
+
let usedOcrLanguage = false;
|
|
94
|
+
let usedPutNotesAtLast = false;
|
|
95
|
+
let usedOutputErrorToConsole = false;
|
|
96
|
+
// Parse the arguments list
|
|
97
|
+
for (let i = 0; i < args.length; i++) {
|
|
98
|
+
const arg = args[i];
|
|
99
|
+
// Help flags trigger immediate termination of parsing and print help
|
|
100
|
+
if (arg === '-h' || arg === '--help') {
|
|
101
|
+
showHelp = true;
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
if (arg.startsWith('--')) {
|
|
105
|
+
let cleanKey;
|
|
106
|
+
let val;
|
|
107
|
+
// Support --key=value syntax
|
|
108
|
+
if (arg.includes('=')) {
|
|
109
|
+
const idx = arg.indexOf('=');
|
|
110
|
+
cleanKey = arg.slice(2, idx);
|
|
111
|
+
val = arg.slice(idx + 1);
|
|
112
|
+
}
|
|
113
|
+
// Support negation shorthand: --no-ocr sets ocr to false
|
|
114
|
+
else if (arg.startsWith('--no-')) {
|
|
115
|
+
cleanKey = arg.slice(5);
|
|
116
|
+
val = 'false';
|
|
117
|
+
}
|
|
118
|
+
// Support space-separated options or bare flags
|
|
119
|
+
else {
|
|
120
|
+
cleanKey = arg.slice(2);
|
|
121
|
+
const isKnownBoolean = knownParserBooleans.has(cleanKey) ||
|
|
122
|
+
knownGeneratorBooleans.has(cleanKey) ||
|
|
123
|
+
cleanKey === 'verbose' ||
|
|
124
|
+
cleanKey === 'toText' ||
|
|
125
|
+
cleanKey === 'outputErrorToConsole';
|
|
126
|
+
const isNextBool = i + 1 < args.length &&
|
|
127
|
+
(args[i + 1].toLowerCase() === 'true' || args[i + 1].toLowerCase() === 'false');
|
|
128
|
+
// If the next arg is not another option flag, treat it as the value (e.g., --to html)
|
|
129
|
+
// But if this is a known boolean, only consume the next argument if it is a valid boolean string.
|
|
130
|
+
if (i + 1 < args.length && !args[i + 1].startsWith('-') && (!isKnownBoolean || isNextBool)) {
|
|
131
|
+
val = args[i + 1];
|
|
132
|
+
i++;
|
|
133
|
+
}
|
|
134
|
+
// Bare presence implies true (e.g., --ocr is true)
|
|
135
|
+
else {
|
|
136
|
+
val = 'true';
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// Parse boolean strings to raw boolean types
|
|
140
|
+
const lowerValue = val.toLowerCase();
|
|
141
|
+
const boolValue = lowerValue === 'true' ? true : (lowerValue === 'false' ? false : undefined);
|
|
142
|
+
// Map core CLI options to variables
|
|
143
|
+
if (cleanKey === 'to') {
|
|
144
|
+
toFlagOption = val;
|
|
145
|
+
}
|
|
146
|
+
else if (cleanKey === 'format') {
|
|
147
|
+
formatFlagOption = val;
|
|
148
|
+
usedFormat = true;
|
|
149
|
+
}
|
|
150
|
+
else if (cleanKey === 'output') {
|
|
151
|
+
outputFile = val;
|
|
152
|
+
}
|
|
153
|
+
else if (cleanKey === 'toText') {
|
|
154
|
+
toTextOption = boolValue !== undefined ? boolValue : true;
|
|
155
|
+
usedToText = true;
|
|
156
|
+
}
|
|
157
|
+
else if (cleanKey === 'verbose') {
|
|
158
|
+
verbose = boolValue !== undefined ? boolValue : true;
|
|
159
|
+
}
|
|
160
|
+
else if (cleanKey === 'ocrLanguage') {
|
|
161
|
+
config.ocrLanguage = val;
|
|
162
|
+
usedOcrLanguage = true;
|
|
163
|
+
}
|
|
164
|
+
else if (cleanKey === 'putNotesAtLast') {
|
|
165
|
+
config.putNotesAtLast = boolValue !== undefined ? boolValue : true;
|
|
166
|
+
usedPutNotesAtLast = true;
|
|
167
|
+
}
|
|
168
|
+
else if (cleanKey === 'outputErrorToConsole') {
|
|
169
|
+
verbose = boolValue !== undefined ? boolValue : true;
|
|
170
|
+
usedOutputErrorToConsole = true;
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
// Check if the flag belongs to generatorConfig or a specific sub-generator (e.g., htmlConfig)
|
|
174
|
+
const isGeneratorOption = knownGeneratorBooleans.has(cleanKey) || generatorPrefixes.some(pref => cleanKey.startsWith(pref));
|
|
175
|
+
const target = isGeneratorOption ? generatorConfig : config;
|
|
176
|
+
let path = cleanKey;
|
|
177
|
+
// Strip generatorConfig prefix to flatten it onto the generatorConfig object
|
|
178
|
+
if (isGeneratorOption && cleanKey.startsWith('generatorConfig.')) {
|
|
179
|
+
path = cleanKey.slice('generatorConfig.'.length);
|
|
180
|
+
}
|
|
181
|
+
// Support nested dot-notation parsing (e.g., --ocrConfig.language=fra)
|
|
182
|
+
if (path.includes('.')) {
|
|
183
|
+
const parts = path.split('.');
|
|
184
|
+
let current = target;
|
|
185
|
+
for (let j = 0; j < parts.length - 1; j++) {
|
|
186
|
+
const part = parts[j];
|
|
187
|
+
if (!current[part])
|
|
188
|
+
current[part] = {};
|
|
189
|
+
current = current[part];
|
|
190
|
+
}
|
|
191
|
+
const lastPart = parts[parts.length - 1];
|
|
192
|
+
current[lastPart] = boolValue !== undefined ? boolValue : val;
|
|
193
|
+
}
|
|
194
|
+
// Flat key assignment
|
|
195
|
+
else {
|
|
196
|
+
if (boolValue !== undefined) {
|
|
197
|
+
target[path] = boolValue;
|
|
198
|
+
}
|
|
199
|
+
else if (knownParserBooleans.has(path) || (isGeneratorOption && knownGeneratorBooleans.has(path))) {
|
|
200
|
+
console.warn(`Invalid boolean value for --${cleanKey}: ${val}. Using default.`);
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
target[path] = val;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
// First positional argument that is not a flag is treated as the input file path
|
|
210
|
+
if (!fileArg) {
|
|
211
|
+
fileArg = arg;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (fileArg && !showHelp) {
|
|
216
|
+
// Resolve output format prioritizing: --to > --format > --toText
|
|
217
|
+
let outputFormat;
|
|
218
|
+
if (toFlagOption) {
|
|
219
|
+
outputFormat = toFlagOption;
|
|
220
|
+
}
|
|
221
|
+
else if (formatFlagOption) {
|
|
222
|
+
outputFormat = formatFlagOption;
|
|
223
|
+
}
|
|
224
|
+
else if (toTextOption === true) {
|
|
225
|
+
outputFormat = 'text';
|
|
226
|
+
}
|
|
227
|
+
// Display warning messages for any deprecated CLI options used
|
|
228
|
+
if (usedFormat) {
|
|
229
|
+
console.warn('Warning: --format is deprecated. Use --to instead.');
|
|
230
|
+
}
|
|
231
|
+
if (usedToText) {
|
|
232
|
+
console.warn('Warning: --toText is deprecated. Use --to=text instead.');
|
|
233
|
+
}
|
|
234
|
+
if (usedOcrLanguage) {
|
|
235
|
+
console.warn('Warning: --ocrLanguage is deprecated. Use --ocrConfig.language instead.');
|
|
236
|
+
}
|
|
237
|
+
if (usedPutNotesAtLast) {
|
|
238
|
+
console.warn('Warning: --putNotesAtLast is deprecated and will be ignored by all parsers.');
|
|
239
|
+
}
|
|
240
|
+
if (usedOutputErrorToConsole) {
|
|
241
|
+
console.warn('Warning: --outputErrorToConsole is deprecated. Use --verbose instead.');
|
|
242
|
+
}
|
|
243
|
+
// Intercept parser warning callbacks to format and print issues when verbose is enabled
|
|
244
|
+
const originalOnWarning = config.onWarning;
|
|
245
|
+
config.onWarning = (issue) => {
|
|
246
|
+
if (verbose) {
|
|
247
|
+
const severity = issue.type === 'error' ? 'Error' : 'Warning';
|
|
248
|
+
console.error(`[OfficeParser ${severity}] [${issue.code}]: ${issue.message}`);
|
|
249
|
+
if (issue.details) {
|
|
250
|
+
console.error(issue.details);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (originalOnWarning)
|
|
254
|
+
originalOnWarning(issue);
|
|
255
|
+
};
|
|
256
|
+
// Propagate newlineDelimiter and csvDelimiter if configured flatly but not in generator sub-configs
|
|
257
|
+
if (config.newlineDelimiter !== undefined) {
|
|
258
|
+
if (!generatorConfig.textConfig)
|
|
259
|
+
generatorConfig.textConfig = {};
|
|
260
|
+
if (generatorConfig.textConfig.newlineDelimiter === undefined) {
|
|
261
|
+
generatorConfig.textConfig.newlineDelimiter = config.newlineDelimiter;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (config.csvDelimiter !== undefined) {
|
|
265
|
+
if (!generatorConfig.csvConfig)
|
|
266
|
+
generatorConfig.csvConfig = {};
|
|
267
|
+
if (generatorConfig.csvConfig.columnDelimiter === undefined) {
|
|
268
|
+
generatorConfig.csvConfig.columnDelimiter = config.csvDelimiter;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
// Run the main parser
|
|
272
|
+
OfficeParser_js_1.OfficeParser.parseOffice(fileArg, config)
|
|
273
|
+
.then(async (ast) => {
|
|
274
|
+
let output;
|
|
275
|
+
// Generate JSON output or convert AST using OfficeGenerator
|
|
276
|
+
if (outputFormat === 'json') {
|
|
277
|
+
output = JSON.stringify(ast, null, 2);
|
|
278
|
+
}
|
|
279
|
+
else if (outputFormat) {
|
|
280
|
+
const result = await OfficeGenerator_js_1.OfficeGenerator.generate(ast, outputFormat, generatorConfig);
|
|
281
|
+
if (Array.isArray(result.value)) {
|
|
282
|
+
output = JSON.stringify(result.value, null, 2);
|
|
283
|
+
}
|
|
284
|
+
else {
|
|
285
|
+
output = result.value;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
else {
|
|
289
|
+
output = JSON.stringify(ast, null, 2);
|
|
290
|
+
}
|
|
291
|
+
// Write generated output to output file or print to standard output
|
|
292
|
+
if (outputFile) {
|
|
293
|
+
if (output instanceof Uint8Array) {
|
|
294
|
+
fs.writeFileSync(outputFile, output);
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
fs.writeFileSync(outputFile, output, 'utf8');
|
|
298
|
+
}
|
|
299
|
+
if (verbose)
|
|
300
|
+
console.log(`Output written to ${outputFile}`);
|
|
301
|
+
}
|
|
302
|
+
else {
|
|
303
|
+
if (output instanceof Uint8Array) {
|
|
304
|
+
process.stdout.write(output);
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
process.stdout.write(output + '\n');
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
// Ensure OCR workers are terminated for clean CLI exit
|
|
311
|
+
if (config.ocr) {
|
|
312
|
+
await OfficeParser_js_1.OfficeParser.terminateOcr();
|
|
313
|
+
}
|
|
314
|
+
})
|
|
315
|
+
.catch(async (err) => {
|
|
316
|
+
// Handle and display parsing error messages
|
|
317
|
+
console.error(`Error parsing file "${fileArg}":`);
|
|
318
|
+
if (verbose) {
|
|
319
|
+
console.error(err);
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
console.error(err.message || err);
|
|
323
|
+
console.error('Use --verbose for full stack trace.');
|
|
324
|
+
}
|
|
325
|
+
// Ensure OCR workers are terminated even on error to prevent process hang
|
|
326
|
+
if (config.ocr) {
|
|
327
|
+
await OfficeParser_js_1.OfficeParser.terminateOcr();
|
|
328
|
+
}
|
|
329
|
+
process.exit(1);
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
else {
|
|
333
|
+
console.log('Usage: officeparser <file> [options]');
|
|
334
|
+
console.log('');
|
|
335
|
+
console.log('Options:');
|
|
336
|
+
console.log(' --to=json|text|md|html|pdf|csv|rtf|epub|chunks Target conversion format (default: json)');
|
|
337
|
+
console.log(' --output=file.ext Save output to file instead of stdout');
|
|
338
|
+
console.log(' --fileType=docx|xlsx|pptx|odt|... Explicitly override input file type detection');
|
|
339
|
+
console.log(' --ocr Enable OCR for images (default: false)');
|
|
340
|
+
console.log(' --ocrConfig.language=eng OCR language (default: eng)');
|
|
341
|
+
console.log(' --extractAttachments Extract embedded attachments (default: false)');
|
|
342
|
+
console.log(' --ignoreNotes Ignore footnotes/endnotes/speaker notes (default: false)');
|
|
343
|
+
console.log(' --ignoreComments Ignore inline comments (default: false)');
|
|
344
|
+
console.log(' --ignoreHeadersAndFooters Ignore headers and footers (default: false)');
|
|
345
|
+
console.log(' --ignoreSlideMasters Ignore slide masters (default: false)');
|
|
346
|
+
console.log(' --ignoreInternalLinks Ignore internal links (default: false)');
|
|
347
|
+
console.log(' --includeRawContent Include raw content in AST (default: false)');
|
|
348
|
+
console.log(' --serializeRawContent Serialize raw XML content (default: true)');
|
|
349
|
+
console.log(' --preserveXmlWhitespace Keep raw formatting space (default: false)');
|
|
350
|
+
console.log(' --includeBreakNodes Include break nodes (DOCX only, default: false)');
|
|
351
|
+
console.log(' --verbose Show full error stack traces and warning logs');
|
|
352
|
+
console.log(' --newlineDelimiter=string Delimiter string between blocks/lines (default: \\n)');
|
|
353
|
+
console.log(' --csvDelimiter=char Custom CSV delimiter (default: ,)');
|
|
354
|
+
console.log(' --htmlParserConfig.preserveIframes Keep non-YouTube <iframe> embeds (dropped by default)');
|
|
355
|
+
console.log('');
|
|
356
|
+
console.log('High-Value Generator Options:');
|
|
357
|
+
console.log(' --includeFormatting Include font formatting like bold/italic (default: true)');
|
|
358
|
+
console.log(' --renderMetadata Render metadata in output content (default: false)');
|
|
359
|
+
console.log(' --htmlConfig.containerWidth=value HTML container width (auto | px | % | vw etc., default: auto)');
|
|
360
|
+
console.log(' --htmlConfig.sourceAttributes Carry each rich node\'s source in a data-* attribute (default: false)');
|
|
361
|
+
console.log('');
|
|
362
|
+
console.log('Advanced Nested Config Examples:');
|
|
363
|
+
console.log(' --pdfConfig.format=Letter Configure Puppeteer PDF format (A4 | Letter | Legal etc.)');
|
|
364
|
+
console.log(' --chunksConfig.strategy=fixed-size Chunking strategy (fixed-size | document-structure | semantic)');
|
|
365
|
+
console.log(' --mdConfig.dialect=github Markdown dialect (extended | github | gitlab | obsidian | pandoc | commonmark)');
|
|
366
|
+
console.log(' --mdConfig.fallbackToHtml=false Disable HTML fallback for unsupported Markdown features (default: true)');
|
|
367
|
+
console.log(' --mdConfig.fallbackToHtml.inlineFormatting Round-trip inline color/highlight/font-size as <span style> (opt-in, default: false)');
|
|
368
|
+
console.log('');
|
|
369
|
+
console.log('Format Syntax:');
|
|
370
|
+
console.log(' Flags can be written as --flag (presence implies true), --no-flag (negation),');
|
|
371
|
+
console.log(' --flag=value, or --flag value.');
|
|
372
|
+
console.log('');
|
|
373
|
+
console.log('Examples:');
|
|
374
|
+
console.log(' officeparser document.docx');
|
|
375
|
+
console.log(' officeparser document.docx --to html --output doc.html');
|
|
376
|
+
console.log(' officeparser document.docx --to md');
|
|
377
|
+
console.log(' officeparser report.pdf --ocr --ocrConfig.language eng --to text');
|
|
378
|
+
console.log(' officeparser data.xlsx --to csv --output data.csv --csvDelimiter ";"');
|
|
379
|
+
console.log(' officeparser document.docx --extractAttachments --to epub --output document.epub');
|
|
380
|
+
console.log(' officeparser image_doc --fileType docx --to json');
|
|
381
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { DeepRequired, DocumentStructureChunkingConfig, FixedSizeChunkingConfig, FullGeneratorConfig, OfficeParserConfig, SemanticChunkingConfig } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The default regex used for identifying sentence boundaries.
|
|
4
|
+
* When this default is used, the generator employs a high-fidelity "robust"
|
|
5
|
+
* segmenter that accounts for common abbreviations (Mr., Dr., etc.).
|
|
6
|
+
*/
|
|
7
|
+
export declare const DEFAULT_SENTENCE_BOUNDARY_REGEX: RegExp;
|
|
8
|
+
/**
|
|
9
|
+
* Common abbreviations that should not trigger a sentence split when followed by a period.
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_ABBREVIATIONS: string[];
|
|
12
|
+
/**
|
|
13
|
+
* Default configuration for the OfficeParser.
|
|
14
|
+
*/
|
|
15
|
+
export declare const DEFAULT_OFFICE_PARSER_CONFIG: DeepRequired<OfficeParserConfig>;
|
|
16
|
+
/**
|
|
17
|
+
* Default configuration for Fixed-Size chunking.
|
|
18
|
+
*/
|
|
19
|
+
export declare const DEFAULT_FIXED_SIZE_CHUNKING_CONFIG: Required<Omit<FixedSizeChunkingConfig, 'embeddingFunction' | 'sentenceBoundaryRegex' | 'abbreviations'>> & {
|
|
20
|
+
sentenceBoundaryRegex: string | RegExp;
|
|
21
|
+
abbreviations: string[];
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Default configuration for Document-Structure chunking.
|
|
25
|
+
*/
|
|
26
|
+
export declare const DEFAULT_DOCUMENT_STRUCTURE_CHUNKING_CONFIG: Required<Omit<DocumentStructureChunkingConfig, 'sentenceBoundaryRegex' | 'abbreviations'>> & {
|
|
27
|
+
sentenceBoundaryRegex: string | RegExp;
|
|
28
|
+
abbreviations: string[];
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Default configuration for Semantic chunking.
|
|
32
|
+
* Note: `embeddingFunction` has no meaningful default and must be provided by the user.
|
|
33
|
+
*/
|
|
34
|
+
export declare const DEFAULT_SEMANTIC_CHUNKING_CONFIG: Required<Omit<SemanticChunkingConfig, 'embeddingFunction' | 'sentenceBoundaryRegex' | 'abbreviations'>> & {
|
|
35
|
+
sentenceBoundaryRegex: string | RegExp;
|
|
36
|
+
abbreviations: string[];
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Default configuration for the OfficeGenerator.
|
|
40
|
+
*/
|
|
41
|
+
export declare const DEFAULT_GENERATOR_CONFIG: FullGeneratorConfig;
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_GENERATOR_CONFIG = exports.DEFAULT_SEMANTIC_CHUNKING_CONFIG = exports.DEFAULT_DOCUMENT_STRUCTURE_CHUNKING_CONFIG = exports.DEFAULT_FIXED_SIZE_CHUNKING_CONFIG = exports.DEFAULT_OFFICE_PARSER_CONFIG = exports.DEFAULT_ABBREVIATIONS = exports.DEFAULT_SENTENCE_BOUNDARY_REGEX = void 0;
|
|
4
|
+
const PDFJS_VERSION = '6.1.200';
|
|
5
|
+
const DEFAULT_PDF_WORKER_SRC = typeof __SLIM__ !== 'undefined' && __SLIM__ ? '' : `https://cdn.jsdelivr.net/npm/pdfjs-dist@${PDFJS_VERSION}/build/pdf.worker.min.mjs`;
|
|
6
|
+
/**
|
|
7
|
+
* The default regex used for identifying sentence boundaries.
|
|
8
|
+
* When this default is used, the generator employs a high-fidelity "robust"
|
|
9
|
+
* segmenter that accounts for common abbreviations (Mr., Dr., etc.).
|
|
10
|
+
*/
|
|
11
|
+
exports.DEFAULT_SENTENCE_BOUNDARY_REGEX = /[.!?。!?]/;
|
|
12
|
+
/**
|
|
13
|
+
* Common abbreviations that should not trigger a sentence split when followed by a period.
|
|
14
|
+
*/
|
|
15
|
+
exports.DEFAULT_ABBREVIATIONS = ['Mr', 'Dr', 'Ms', 'Inc', 'Ltd', 'Prof', 'Sr', 'Jr', 'vs', 'etc'];
|
|
16
|
+
/** Default timeout values for OCR */
|
|
17
|
+
const DEFAULT_OCR_TIMEOUT = {
|
|
18
|
+
autoTerminate: 10000,
|
|
19
|
+
workerLoad: 60000,
|
|
20
|
+
recognition: 30000,
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Default configuration for OCR.
|
|
24
|
+
*/
|
|
25
|
+
const DEFAULT_OCR_CONFIG = {
|
|
26
|
+
language: 'eng',
|
|
27
|
+
workerPath: '',
|
|
28
|
+
corePath: '',
|
|
29
|
+
langPath: '',
|
|
30
|
+
// Preferred: consolidated timeout object. New code should always read from here.
|
|
31
|
+
timeout: DEFAULT_OCR_TIMEOUT,
|
|
32
|
+
// Kept for backward compatibility. When timeout.autoTerminate is set (as above),
|
|
33
|
+
// the ocrUtils resolution logic will prefer timeout.autoTerminate over this flat field.
|
|
34
|
+
autoTerminateTimeout: DEFAULT_OCR_TIMEOUT.autoTerminate,
|
|
35
|
+
abortSignal: null,
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Default configuration for HTML/XHTML parsing. `preserveAttributes` is off so that the AST is
|
|
39
|
+
* byte-identical to previous releases unless a caller opts in - see `HtmlParserConfig`.
|
|
40
|
+
*/
|
|
41
|
+
const DEFAULT_HTML_PARSER_CONFIG = {
|
|
42
|
+
preserveAttributes: false,
|
|
43
|
+
preserveIframes: false,
|
|
44
|
+
embedFolkForms: false,
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* Default configuration for the OfficeParser.
|
|
48
|
+
*/
|
|
49
|
+
exports.DEFAULT_OFFICE_PARSER_CONFIG = {
|
|
50
|
+
outputErrorToConsole: false,
|
|
51
|
+
onWarning: () => { },
|
|
52
|
+
newlineDelimiter: '\n',
|
|
53
|
+
ignoreNotes: false,
|
|
54
|
+
ignoreComments: false,
|
|
55
|
+
ignoreHeadersAndFooters: false,
|
|
56
|
+
ignoreSlideMasters: false,
|
|
57
|
+
putNotesAtLast: false,
|
|
58
|
+
extractAttachments: false,
|
|
59
|
+
includeRawContent: false,
|
|
60
|
+
ocr: false,
|
|
61
|
+
ocrLanguage: 'eng',
|
|
62
|
+
ocrConfig: DEFAULT_OCR_CONFIG,
|
|
63
|
+
abortSignal: null,
|
|
64
|
+
serializeRawContent: true,
|
|
65
|
+
preserveXmlWhitespace: false,
|
|
66
|
+
pdfWorkerSrc: DEFAULT_PDF_WORKER_SRC,
|
|
67
|
+
includeBreakNodes: false,
|
|
68
|
+
ignoreInternalLinks: false,
|
|
69
|
+
fileType: null,
|
|
70
|
+
csvDelimiter: ',',
|
|
71
|
+
decompressionLimits: {
|
|
72
|
+
maxUncompressedBytes: 512 * 1024 * 1024,
|
|
73
|
+
maxZipEntries: 10000,
|
|
74
|
+
maxTableCells: 1000000,
|
|
75
|
+
},
|
|
76
|
+
htmlParserConfig: DEFAULT_HTML_PARSER_CONFIG,
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Default configuration for HTML generation.
|
|
80
|
+
*/
|
|
81
|
+
const DEFAULT_HTML_GENERATOR_CONFIG = {
|
|
82
|
+
standalone: true,
|
|
83
|
+
chartJsSrc: typeof __SLIM__ !== 'undefined' && __SLIM__ ? '' : 'https://cdn.jsdelivr.net/npm/chart.js',
|
|
84
|
+
containerWidth: 'auto',
|
|
85
|
+
customCss: '',
|
|
86
|
+
injections: {
|
|
87
|
+
headStart: '',
|
|
88
|
+
headEnd: '',
|
|
89
|
+
bodyStart: '',
|
|
90
|
+
bodyEnd: '',
|
|
91
|
+
},
|
|
92
|
+
sourceAttributes: false,
|
|
93
|
+
gatedEmbeds: false,
|
|
94
|
+
};
|
|
95
|
+
/**
|
|
96
|
+
* Default configuration for PDF generation.
|
|
97
|
+
*/
|
|
98
|
+
const DEFAULT_PDF_GENERATOR_CONFIG = {
|
|
99
|
+
format: 'A4',
|
|
100
|
+
width: '',
|
|
101
|
+
height: '',
|
|
102
|
+
landscape: false,
|
|
103
|
+
printBackground: true,
|
|
104
|
+
scale: 1,
|
|
105
|
+
margin: {
|
|
106
|
+
top: 0,
|
|
107
|
+
right: 0,
|
|
108
|
+
bottom: 0,
|
|
109
|
+
left: 0
|
|
110
|
+
},
|
|
111
|
+
displayHeaderFooter: false,
|
|
112
|
+
headerTemplate: '',
|
|
113
|
+
footerTemplate: '',
|
|
114
|
+
launchOptions: {
|
|
115
|
+
headless: true,
|
|
116
|
+
args: ['--no-sandbox', '--disable-setuid-sandbox']
|
|
117
|
+
},
|
|
118
|
+
timeout: 30000,
|
|
119
|
+
};
|
|
120
|
+
/**
|
|
121
|
+
* Default configuration for CSV generation.
|
|
122
|
+
*/
|
|
123
|
+
const DEFAULT_CSV_GENERATOR_CONFIG = {
|
|
124
|
+
sheets: '',
|
|
125
|
+
mergeSheets: true,
|
|
126
|
+
columnDelimiter: ',',
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Default configuration for Markdown generation.
|
|
130
|
+
*/
|
|
131
|
+
const DEFAULT_MD_GENERATOR_CONFIG = {
|
|
132
|
+
fallbackToHtml: true,
|
|
133
|
+
dialect: 'extended',
|
|
134
|
+
};
|
|
135
|
+
/**
|
|
136
|
+
* Default configuration for plain text generation.
|
|
137
|
+
*/
|
|
138
|
+
const DEFAULT_TEXT_GENERATOR_CONFIG = {
|
|
139
|
+
newlineDelimiter: '\n',
|
|
140
|
+
preserveLayout: true,
|
|
141
|
+
renderNotes: true,
|
|
142
|
+
};
|
|
143
|
+
/**
|
|
144
|
+
* Default configuration for Fixed-Size chunking.
|
|
145
|
+
*/
|
|
146
|
+
exports.DEFAULT_FIXED_SIZE_CHUNKING_CONFIG = {
|
|
147
|
+
strategy: 'fixed-size',
|
|
148
|
+
chunkSize: 1000,
|
|
149
|
+
chunkOverlap: 200,
|
|
150
|
+
separators: ['\n\n', '\n', ' ', ''],
|
|
151
|
+
stripWhitespace: true,
|
|
152
|
+
includeMetadata: true,
|
|
153
|
+
addStartIndex: false,
|
|
154
|
+
lengthFunction: (text) => text.length,
|
|
155
|
+
sentenceBoundaryRegex: exports.DEFAULT_SENTENCE_BOUNDARY_REGEX,
|
|
156
|
+
abbreviations: exports.DEFAULT_ABBREVIATIONS,
|
|
157
|
+
};
|
|
158
|
+
/**
|
|
159
|
+
* Default configuration for Document-Structure chunking.
|
|
160
|
+
*/
|
|
161
|
+
exports.DEFAULT_DOCUMENT_STRUCTURE_CHUNKING_CONFIG = {
|
|
162
|
+
strategy: 'document-structure',
|
|
163
|
+
splitBy: 'paragraph',
|
|
164
|
+
maxChunkSize: 1000,
|
|
165
|
+
tableSplitStrategy: 'row',
|
|
166
|
+
stripWhitespace: true,
|
|
167
|
+
includeMetadata: true,
|
|
168
|
+
addStartIndex: false,
|
|
169
|
+
lengthFunction: (text) => text.length,
|
|
170
|
+
sentenceBoundaryRegex: exports.DEFAULT_SENTENCE_BOUNDARY_REGEX,
|
|
171
|
+
abbreviations: exports.DEFAULT_ABBREVIATIONS,
|
|
172
|
+
};
|
|
173
|
+
/**
|
|
174
|
+
* Default configuration for Semantic chunking.
|
|
175
|
+
* Note: `embeddingFunction` has no meaningful default and must be provided by the user.
|
|
176
|
+
*/
|
|
177
|
+
exports.DEFAULT_SEMANTIC_CHUNKING_CONFIG = {
|
|
178
|
+
strategy: 'semantic',
|
|
179
|
+
similarityThreshold: 0.8,
|
|
180
|
+
maxChunkSize: 2000,
|
|
181
|
+
bufferSize: 1,
|
|
182
|
+
embeddingBatchSize: 50,
|
|
183
|
+
stripWhitespace: true,
|
|
184
|
+
includeMetadata: true,
|
|
185
|
+
addStartIndex: false,
|
|
186
|
+
lengthFunction: (text) => text.length,
|
|
187
|
+
sentenceBoundaryRegex: exports.DEFAULT_SENTENCE_BOUNDARY_REGEX,
|
|
188
|
+
abbreviations: exports.DEFAULT_ABBREVIATIONS,
|
|
189
|
+
timeout: 10000,
|
|
190
|
+
};
|
|
191
|
+
/**
|
|
192
|
+
* The resolved default chunking config (uses document-structure as default strategy).
|
|
193
|
+
*/
|
|
194
|
+
const DEFAULT_CHUNKING_CONFIG = exports.DEFAULT_DOCUMENT_STRUCTURE_CHUNKING_CONFIG;
|
|
195
|
+
/**
|
|
196
|
+
* Default configuration for the OfficeGenerator.
|
|
197
|
+
*/
|
|
198
|
+
exports.DEFAULT_GENERATOR_CONFIG = {
|
|
199
|
+
onNode: () => { },
|
|
200
|
+
onWarning: () => { },
|
|
201
|
+
styleMap: [],
|
|
202
|
+
includeFormatting: true,
|
|
203
|
+
generateIds: true,
|
|
204
|
+
renderMetadata: false,
|
|
205
|
+
metadataOverrides: {},
|
|
206
|
+
ignoreDefaultStyleMap: false,
|
|
207
|
+
includeImages: true,
|
|
208
|
+
includeCharts: true,
|
|
209
|
+
ignoreInternalLinks: false,
|
|
210
|
+
abortSignal: null,
|
|
211
|
+
htmlConfig: DEFAULT_HTML_GENERATOR_CONFIG,
|
|
212
|
+
mdConfig: DEFAULT_MD_GENERATOR_CONFIG,
|
|
213
|
+
pdfConfig: DEFAULT_PDF_GENERATOR_CONFIG,
|
|
214
|
+
csvConfig: DEFAULT_CSV_GENERATOR_CONFIG,
|
|
215
|
+
textConfig: DEFAULT_TEXT_GENERATOR_CONFIG,
|
|
216
|
+
rtfConfig: {},
|
|
217
|
+
chunksConfig: DEFAULT_CHUNKING_CONFIG,
|
|
218
|
+
};
|