@gmickel/gno 1.46.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (235) hide show
  1. package/README.md +17 -5
  2. package/THIRD_PARTY_NOTICES.md +46 -0
  3. package/assets/skill/SKILL.md +7 -6
  4. package/assets/spa-production.json.gz +0 -0
  5. package/browser-extension/artifacts/gno-browser-clipper-v2.1.0.zip +0 -0
  6. package/browser-extension/artifacts/gno-browser-clipper-v2.1.0.zip.sha256 +1 -0
  7. package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
  8. package/browser-extension/dist/manifest.json +1 -1
  9. package/browser-extension/dist/preview.html +1 -1
  10. package/browser-extension/dist/service-worker.js +32 -33
  11. package/bunfig.toml +2 -0
  12. package/package.json +40 -26
  13. package/spec/cli.md +29 -4
  14. package/spec/db/schema.sql +146 -1
  15. package/spec/mcp.md +26 -0
  16. package/src/app/context-runtime-types.ts +3 -0
  17. package/src/app/context-runtime.ts +2 -0
  18. package/src/cli/commands/ask.ts +6 -1
  19. package/src/cli/commands/daemon.ts +21 -8
  20. package/src/cli/commands/embed.ts +77 -41
  21. package/src/cli/detach.ts +3 -2
  22. package/src/config/types.ts +3 -3
  23. package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
  24. package/src/converters/adapters/officeparser/adapter.ts +1 -2
  25. package/src/converters/versions.ts +6 -8
  26. package/src/core/context-evidence.ts +8 -4
  27. package/src/core/job-manager.ts +95 -13
  28. package/src/core/network-boundary-inventory.ts +10 -0
  29. package/src/core/shutdown-budget.ts +45 -0
  30. package/src/embed/backlog.ts +107 -4
  31. package/src/embed/batch.ts +42 -2
  32. package/src/embed/fingerprint.ts +16 -0
  33. package/src/embed/retry.ts +113 -5
  34. package/src/embed/variant-backlog.ts +105 -0
  35. package/src/embed/variant-plan.ts +62 -0
  36. package/src/embed/variant-retry.ts +113 -0
  37. package/src/ingestion/graph-reconciliation.ts +327 -0
  38. package/src/ingestion/sync.ts +9 -272
  39. package/src/llm/http-inference.ts +6 -0
  40. package/src/llm/httpEmbedding.ts +37 -6
  41. package/src/llm/httpGeneration.ts +18 -3
  42. package/src/llm/httpRerank.ts +23 -5
  43. package/src/llm/inference-cancellation.ts +168 -0
  44. package/src/llm/inference-scope.ts +202 -0
  45. package/src/llm/lazy-ports.ts +115 -0
  46. package/src/llm/native-worker/client.ts +541 -0
  47. package/src/llm/native-worker/dispatcher.ts +228 -0
  48. package/src/llm/native-worker/embedding-identity.ts +33 -0
  49. package/src/llm/native-worker/entry.ts +173 -0
  50. package/src/llm/native-worker/errors.ts +32 -0
  51. package/src/llm/native-worker/evaluation.ts +16 -0
  52. package/src/llm/native-worker/owned-exit.ts +108 -0
  53. package/src/llm/native-worker/owner.ts +141 -0
  54. package/src/llm/native-worker/ports.ts +317 -0
  55. package/src/llm/native-worker/protocol.ts +442 -0
  56. package/src/llm/native-worker/runtime-config.ts +92 -0
  57. package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
  58. package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
  59. package/src/llm/nodeLlamaCpp/generation.ts +34 -5
  60. package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
  61. package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
  62. package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
  63. package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
  64. package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
  65. package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
  66. package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
  67. package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
  68. package/src/llm/types.ts +35 -5
  69. package/src/mcp/context.ts +27 -0
  70. package/src/mcp/http-transport.ts +12 -10
  71. package/src/mcp/server.ts +3 -0
  72. package/src/mcp/tool-profile.ts +30 -8
  73. package/src/mcp/tools/context.ts +8 -11
  74. package/src/mcp/tools/embed.ts +1 -1
  75. package/src/mcp/tools/index-cmd.ts +1 -1
  76. package/src/mcp/tools/index.ts +10 -8
  77. package/src/mcp/tools/query.ts +14 -30
  78. package/src/mcp/tools/vsearch.ts +1 -1
  79. package/src/pipeline/answer.ts +23 -3
  80. package/src/pipeline/claim-verifier.ts +6 -0
  81. package/src/pipeline/expansion.ts +43 -40
  82. package/src/pipeline/explain.ts +6 -2
  83. package/src/pipeline/filters.ts +63 -0
  84. package/src/pipeline/fusion.ts +29 -9
  85. package/src/pipeline/graph-retrieval.ts +29 -9
  86. package/src/pipeline/hybrid.ts +198 -55
  87. package/src/pipeline/hydration.ts +161 -0
  88. package/src/pipeline/owner-fusion.ts +87 -0
  89. package/src/pipeline/rerank.ts +35 -11
  90. package/src/pipeline/search.ts +13 -2
  91. package/src/pipeline/types.ts +5 -3
  92. package/src/pipeline/vsearch.ts +87 -7
  93. package/src/sdk/client.ts +47 -3
  94. package/src/sdk/embed.ts +63 -39
  95. package/src/serve/background-runtime.ts +1 -1
  96. package/src/serve/context.ts +41 -56
  97. package/src/serve/embed-scheduler.ts +58 -35
  98. package/src/serve/public/components/IndexingProgress.tsx +46 -60
  99. package/src/serve/public/components/PublishExportDialog.tsx +266 -0
  100. package/src/serve/public/globals.built.css +1 -1
  101. package/src/serve/public/globals.css +35 -0
  102. package/src/serve/public/lib/publish-export.ts +81 -1
  103. package/src/serve/public/lib/shiki-language-ids.ts +14 -0
  104. package/src/serve/public/pages/Collections.tsx +12 -46
  105. package/src/serve/public/pages/DocView.tsx +14 -52
  106. package/src/serve/resident-admission.ts +36 -36
  107. package/src/serve/resident-background-work.ts +20 -2
  108. package/src/serve/resident-request.ts +11 -5
  109. package/src/serve/resident-runtime.ts +97 -61
  110. package/src/serve/resident-shutdown.ts +153 -0
  111. package/src/serve/routes/api.ts +3 -1
  112. package/src/serve/server.ts +47 -26
  113. package/src/store/migrations/028-vector-variants.ts +54 -0
  114. package/src/store/migrations/029-graph-reference-state.ts +77 -0
  115. package/src/store/migrations/index.ts +4 -0
  116. package/src/store/sqlite/adapter.ts +251 -183
  117. package/src/store/sqlite/eligibility.ts +174 -0
  118. package/src/store/sqlite/graph-edge-application.ts +66 -0
  119. package/src/store/sqlite/graph-reference-state.ts +194 -0
  120. package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
  121. package/src/store/types.ts +80 -12
  122. package/src/store/vector/eligibility.ts +36 -0
  123. package/src/store/vector/freshness.ts +33 -6
  124. package/src/store/vector/lazy.ts +81 -0
  125. package/src/store/vector/sqlite-vec.ts +106 -54
  126. package/src/store/vector/stats.ts +14 -3
  127. package/src/store/vector/types.ts +35 -2
  128. package/src/store/vector/variant-search.ts +192 -0
  129. package/src/store/vector/variants.ts +451 -0
  130. package/vendor/converters/markitdown-ts/LICENSE +21 -0
  131. package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
  132. package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
  133. package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
  134. package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
  135. package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
  136. package/vendor/converters/markitdown-ts/package.json +77 -0
  137. package/vendor/converters/officeparser/LICENSE +21 -0
  138. package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
  139. package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
  140. package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
  141. package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
  142. package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
  143. package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
  144. package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
  145. package/vendor/converters/officeparser/dist/cli.js +381 -0
  146. package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
  147. package/vendor/converters/officeparser/dist/defaults.js +218 -0
  148. package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
  149. package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
  150. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
  151. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
  152. package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
  153. package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
  154. package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
  155. package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
  156. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
  157. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
  158. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
  159. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
  160. package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
  161. package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
  162. package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
  163. package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
  164. package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
  165. package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
  166. package/vendor/converters/officeparser/dist/index.d.ts +60 -0
  167. package/vendor/converters/officeparser/dist/index.js +72 -0
  168. package/vendor/converters/officeparser/dist/index.mjs +18 -0
  169. package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
  170. package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
  171. package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
  172. package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
  173. package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
  174. package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
  175. package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
  176. package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
  177. package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
  178. package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
  179. package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
  180. package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
  181. package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
  182. package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
  183. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
  184. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
  185. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
  186. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
  187. package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
  188. package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
  189. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
  190. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
  191. package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
  192. package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
  193. package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
  194. package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
  195. package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
  196. package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
  197. package/vendor/converters/officeparser/dist/types.js +107 -0
  198. package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
  199. package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
  200. package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
  201. package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
  202. package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
  203. package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
  204. package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
  205. package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
  206. package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
  207. package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
  208. package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
  209. package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
  210. package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
  211. package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
  212. package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
  213. package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
  214. package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
  215. package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
  216. package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
  217. package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
  218. package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
  219. package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
  220. package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
  221. package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
  222. package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
  223. package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
  224. package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
  225. package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
  226. package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
  227. package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
  228. package/vendor/converters/officeparser/package.json +147 -0
  229. package/vendor/converters/upstream-manifest.json +124 -0
  230. package/vendor/dependency-fixes/README.md +77 -0
  231. package/vendor/dependency-fixes/vendor-converters.py +83 -0
  232. package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip +0 -0
  233. package/browser-extension/artifacts/gno-browser-clipper-v1.46.0.zip.sha256 +0 -1
  234. package/browser-extension/dist/chunk-627emwpj.js +0 -75
  235. /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
@@ -0,0 +1,67 @@
1
+ /**
2
+ * PDF Parser
3
+ *
4
+ * Extracts text, metadata, images, links, and attachments from PDF files using PDF.js (pdfjs-dist).
5
+ *
6
+ * **Features:**
7
+ * - Text extraction with formatting (bold, italic, font, size)
8
+ * - Comprehensive metadata extraction (title, author, subject, creator, producer, creation/modification dates)
9
+ * - Hyperlink extraction from PDF annotations
10
+ * - Heading detection via font size heuristics
11
+ * - Image extraction as attachments with optional OCR (using Tesseract.js)
12
+ * - Embedded file attachment extraction
13
+ * - Layout preservation (respects order of text and images)
14
+ *
15
+ * **PDF Format Limitations (compared to DOCX/ODT):**
16
+ *
17
+ * PDF was designed as a "page description language" for visual fidelity, not semantic structure.
18
+ * The following features **cannot be reliably extracted** from PDFs:
19
+ *
20
+ * - **Tables**: PDF has no table structure. Tables are just text positioned to look tabular.
21
+ * Extracting tables would require complex spatial analysis with many false positives.
22
+ * See: https://stackoverflow.com/questions/36978446/why-is-it-difficult-to-extract-data-from-pdfs
23
+ *
24
+ * - **Lists**: PDF has no list structure. Bullets/numbers are just text characters.
25
+ * No hierarchy or list type information is stored. Would require heuristic detection
26
+ * that would have many edge cases and errors.
27
+ *
28
+ * - **Styles**: PDF has no style definitions like "Heading1" or "Normal". Only visual
29
+ * properties (font, size) exist. We use font size heuristics to detect headings.
30
+ *
31
+ * - **Notes (Footnotes/Endnotes)**: PDF has no concept of footnotes/endnotes as structured
32
+ * elements. They're just smaller text at the bottom of pages.
33
+ *
34
+ * - **Text Color**: While PDF stores color, pdfjs-dist doesn't expose text color in the
35
+ * textContent API. Would require parsing the operator stream which is complex.
36
+ *
37
+ * - **Background Color**: Same limitation as text color.
38
+ *
39
+ * - **Underline/Strikethrough**: These are drawn as separate line elements in PDF,
40
+ * not properties of text. Association would require spatial analysis.
41
+ *
42
+ * **Parsing Approach:**
43
+ * 1. Load PDF document using pdfjs-dist.
44
+ * 2. Extract global metadata from document info dictionary.
45
+ * 3. Extract embedded file attachments.
46
+ * 4. Iterate through each page:
47
+ * a. Collect text items with position and formatting.
48
+ * b. Collect link annotations with associated text.
49
+ * c. Collect images from the operator list.
50
+ * d. Sort all items by vertical position (top-to-bottom reading order).
51
+ * e. Group text into paragraphs/headings based on line breaks and font sizes.
52
+ * f. Process images as attachments with optional OCR.
53
+ * 5. Apply heading detection based on font size heuristics.
54
+ *
55
+ * @module PdfParser
56
+ * @see https://mozilla.github.io/pdf.js/ PDF.js documentation
57
+ * @see https://www.adobe.com/devnet/pdf/pdf_reference.html PDF Reference
58
+ */
59
+ import { FullOfficeParserConfig, OfficeParserAST } from '../types.js';
60
+ /**
61
+ * Parses a PDF file and extracts content.
62
+ *
63
+ * @param buffer - The PDF file buffer
64
+ * @param config - Parser configuration
65
+ * @returns Promise resolving to the parsed AST
66
+ */
67
+ export declare const parsePdf: (buffer: Buffer, config: FullOfficeParserConfig) => Promise<OfficeParserAST>;