@gmickel/gno 1.46.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +21 -4
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/detach.ts +3 -2
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF Parser
|
|
3
|
+
*
|
|
4
|
+
* Extracts text, metadata, images, links, and attachments from PDF files using PDF.js (pdfjs-dist).
|
|
5
|
+
*
|
|
6
|
+
* **Features:**
|
|
7
|
+
* - Text extraction with formatting (bold, italic, font, size)
|
|
8
|
+
* - Comprehensive metadata extraction (title, author, subject, creator, producer, creation/modification dates)
|
|
9
|
+
* - Hyperlink extraction from PDF annotations
|
|
10
|
+
* - Heading detection via font size heuristics
|
|
11
|
+
* - Image extraction as attachments with optional OCR (using Tesseract.js)
|
|
12
|
+
* - Embedded file attachment extraction
|
|
13
|
+
* - Layout preservation (respects order of text and images)
|
|
14
|
+
*
|
|
15
|
+
* **PDF Format Limitations (compared to DOCX/ODT):**
|
|
16
|
+
*
|
|
17
|
+
* PDF was designed as a "page description language" for visual fidelity, not semantic structure.
|
|
18
|
+
* The following features **cannot be reliably extracted** from PDFs:
|
|
19
|
+
*
|
|
20
|
+
* - **Tables**: PDF has no table structure. Tables are just text positioned to look tabular.
|
|
21
|
+
* Extracting tables would require complex spatial analysis with many false positives.
|
|
22
|
+
* See: https://stackoverflow.com/questions/36978446/why-is-it-difficult-to-extract-data-from-pdfs
|
|
23
|
+
*
|
|
24
|
+
* - **Lists**: PDF has no list structure. Bullets/numbers are just text characters.
|
|
25
|
+
* No hierarchy or list type information is stored. Would require heuristic detection
|
|
26
|
+
* that would have many edge cases and errors.
|
|
27
|
+
*
|
|
28
|
+
* - **Styles**: PDF has no style definitions like "Heading1" or "Normal". Only visual
|
|
29
|
+
* properties (font, size) exist. We use font size heuristics to detect headings.
|
|
30
|
+
*
|
|
31
|
+
* - **Notes (Footnotes/Endnotes)**: PDF has no concept of footnotes/endnotes as structured
|
|
32
|
+
* elements. They're just smaller text at the bottom of pages.
|
|
33
|
+
*
|
|
34
|
+
* - **Text Color**: While PDF stores color, pdfjs-dist doesn't expose text color in the
|
|
35
|
+
* textContent API. Would require parsing the operator stream which is complex.
|
|
36
|
+
*
|
|
37
|
+
* - **Background Color**: Same limitation as text color.
|
|
38
|
+
*
|
|
39
|
+
* - **Underline/Strikethrough**: These are drawn as separate line elements in PDF,
|
|
40
|
+
* not properties of text. Association would require spatial analysis.
|
|
41
|
+
*
|
|
42
|
+
* **Parsing Approach:**
|
|
43
|
+
* 1. Load PDF document using pdfjs-dist.
|
|
44
|
+
* 2. Extract global metadata from document info dictionary.
|
|
45
|
+
* 3. Extract embedded file attachments.
|
|
46
|
+
* 4. Iterate through each page:
|
|
47
|
+
* a. Collect text items with position and formatting.
|
|
48
|
+
* b. Collect link annotations with associated text.
|
|
49
|
+
* c. Collect images from the operator list.
|
|
50
|
+
* d. Sort all items by vertical position (top-to-bottom reading order).
|
|
51
|
+
* e. Group text into paragraphs/headings based on line breaks and font sizes.
|
|
52
|
+
* f. Process images as attachments with optional OCR.
|
|
53
|
+
* 5. Apply heading detection based on font size heuristics.
|
|
54
|
+
*
|
|
55
|
+
* @module PdfParser
|
|
56
|
+
* @see https://mozilla.github.io/pdf.js/ PDF.js documentation
|
|
57
|
+
* @see https://www.adobe.com/devnet/pdf/pdf_reference.html PDF Reference
|
|
58
|
+
*/
|
|
59
|
+
import { FullOfficeParserConfig, OfficeParserAST } from '../types.js';
|
|
60
|
+
/**
|
|
61
|
+
* Parses a PDF file and extracts content.
|
|
62
|
+
*
|
|
63
|
+
* @param buffer - The PDF file buffer
|
|
64
|
+
* @param config - Parser configuration
|
|
65
|
+
* @returns Promise resolving to the parsed AST
|
|
66
|
+
*/
|
|
67
|
+
export declare const parsePdf: (buffer: Buffer, config: FullOfficeParserConfig) => Promise<OfficeParserAST>;
|