omni-viewer-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +80 -0
- package/THIRD_PARTY_NOTICES.md +49 -0
- package/dist/assets/audio-engine/audio_engine.mjs +2 -0
- package/dist/assets/audio-engine/audio_engine.wasm +0 -0
- package/dist/assets/pdfjs/pdf.worker.min.mjs +21 -0
- package/dist/host/index.d.ts +133 -0
- package/dist/host/index.js +10 -0
- package/dist/i18n/catalog.en.d.ts +1 -0
- package/dist/i18n/catalog.en.js +411 -0
- package/dist/i18n/catalog.ko.d.ts +1 -0
- package/dist/i18n/catalog.ko.js +68 -0
- package/dist/i18n/index.d.ts +18 -0
- package/dist/i18n/index.js +41 -0
- package/dist/licenses/emscripten.txt +101 -0
- package/dist/licenses/kissfft.txt +28 -0
- package/dist/licenses/pdfjs-dist.txt +177 -0
- package/dist/parsers/a2l/index.d.ts +1 -0
- package/dist/parsers/a2l/index.js +1 -0
- package/dist/parsers/archive/index.d.ts +22 -0
- package/dist/parsers/archive/index.js +46 -0
- package/dist/parsers/archive/model.d.ts +29 -0
- package/dist/parsers/archive/model.js +1 -0
- package/dist/parsers/archive/worker.d.ts +21 -0
- package/dist/parsers/archive/worker.js +1 -0
- package/dist/parsers/arxml/index.d.ts +1 -0
- package/dist/parsers/arxml/index.js +1 -0
- package/dist/parsers/asc/index.d.ts +1 -0
- package/dist/parsers/asc/index.js +1 -0
- package/dist/parsers/audio/index.d.ts +9 -0
- package/dist/parsers/audio/index.js +46 -0
- package/dist/parsers/automotive/index.d.ts +33 -0
- package/dist/parsers/automotive/index.js +973 -0
- package/dist/parsers/blf/index.d.ts +1 -0
- package/dist/parsers/blf/index.js +1 -0
- package/dist/parsers/csv/delimiter.d.ts +31 -0
- package/dist/parsers/csv/delimiter.js +151 -0
- package/dist/parsers/csv/index.d.ts +30 -0
- package/dist/parsers/csv/index.js +108 -0
- package/dist/parsers/csv/parseText.d.ts +20 -0
- package/dist/parsers/csv/parseText.js +108 -0
- package/dist/parsers/csv/sort.d.ts +26 -0
- package/dist/parsers/csv/sort.js +224 -0
- package/dist/parsers/csv/statistics.d.ts +49 -0
- package/dist/parsers/csv/statistics.js +150 -0
- package/dist/parsers/dbc/index.d.ts +56 -0
- package/dist/parsers/dbc/index.js +94 -0
- package/dist/parsers/doc-binary/index.d.ts +35 -0
- package/dist/parsers/doc-binary/index.js +267 -0
- package/dist/parsers/doc-binary/vscode-parser.d.ts +142 -0
- package/dist/parsers/doc-binary/vscode-parser.js +3747 -0
- package/dist/parsers/excel/index.d.ts +17 -0
- package/dist/parsers/excel/index.js +406 -0
- package/dist/parsers/excel/model.d.ts +80 -0
- package/dist/parsers/excel/model.js +48 -0
- package/dist/parsers/excel/serialize.d.ts +3 -0
- package/dist/parsers/excel/serialize.js +78 -0
- package/dist/parsers/hdf5/index.d.ts +75 -0
- package/dist/parsers/hdf5/index.js +658 -0
- package/dist/parsers/hdf5/node.d.ts +21 -0
- package/dist/parsers/hdf5/node.js +95 -0
- package/dist/parsers/json/index.d.ts +15 -0
- package/dist/parsers/json/index.js +102 -0
- package/dist/parsers/json/model.d.ts +42 -0
- package/dist/parsers/json/model.js +140 -0
- package/dist/parsers/json/parse.d.ts +22 -0
- package/dist/parsers/json/parse.js +279 -0
- package/dist/parsers/json/tokenizer.d.ts +17 -0
- package/dist/parsers/json/tokenizer.js +175 -0
- package/dist/parsers/jsonl/index.d.ts +14 -0
- package/dist/parsers/jsonl/index.js +18 -0
- package/dist/parsers/jsonl/model.d.ts +23 -0
- package/dist/parsers/jsonl/model.js +1 -0
- package/dist/parsers/jsonl/parse.d.ts +7 -0
- package/dist/parsers/jsonl/parse.js +34 -0
- package/dist/parsers/markdown/index.d.ts +37 -0
- package/dist/parsers/markdown/index.js +59 -0
- package/dist/parsers/mat/index.d.ts +31 -0
- package/dist/parsers/mat/index.js +426 -0
- package/dist/parsers/mermaid/index.d.ts +5 -0
- package/dist/parsers/mermaid/index.js +3 -0
- package/dist/parsers/mf4/index.d.ts +1 -0
- package/dist/parsers/mf4/index.js +1 -0
- package/dist/parsers/parquet/index.d.ts +21 -0
- package/dist/parsers/parquet/index.js +68 -0
- package/dist/parsers/pcap/index.d.ts +1 -0
- package/dist/parsers/pcap/index.js +1 -0
- package/dist/parsers/pcapng/index.d.ts +1 -0
- package/dist/parsers/pcapng/index.js +1 -0
- package/dist/parsers/plantuml/index.d.ts +5 -0
- package/dist/parsers/plantuml/index.js +4 -0
- package/dist/parsers/ppt-binary/index.d.ts +5 -0
- package/dist/parsers/ppt-binary/index.js +62 -0
- package/dist/parsers/ppt-binary/pptBinaryBuildContext.d.ts +12 -0
- package/dist/parsers/ppt-binary/pptBinaryBuildContext.js +40 -0
- package/dist/parsers/ppt-binary/pptBinaryContainer.d.ts +4 -0
- package/dist/parsers/ppt-binary/pptBinaryContainer.js +174 -0
- package/dist/parsers/ppt-binary/pptBinaryTypes.d.ts +125 -0
- package/dist/parsers/ppt-binary/pptBinaryTypes.js +1 -0
- package/dist/parsers/ppt-binary/pptSlideLayouts.d.ts +9 -0
- package/dist/parsers/ppt-binary/pptSlideLayouts.js +510 -0
- package/dist/parsers/ppt-binary/vscode-parser.d.ts +116 -0
- package/dist/parsers/ppt-binary/vscode-parser.js +2853 -0
- package/dist/parsers/ppt-binary/vscode.d.ts +4 -0
- package/dist/parsers/ppt-binary/vscode.js +12 -0
- package/dist/parsers/pptx/index.d.ts +15 -0
- package/dist/parsers/pptx/index.js +88 -0
- package/dist/parsers/pptx/vscode-parser.d.ts +167 -0
- package/dist/parsers/pptx/vscode-parser.js +1503 -0
- package/dist/parsers/pptx/vscode.d.ts +4 -0
- package/dist/parsers/pptx/vscode.js +18 -0
- package/dist/parsers/pptx/zip-reader.d.ts +3 -0
- package/dist/parsers/pptx/zip-reader.js +20 -0
- package/dist/parsers/proto/index.d.ts +90 -0
- package/dist/parsers/proto/index.js +197 -0
- package/dist/parsers/psd/index.d.ts +16 -0
- package/dist/parsers/psd/index.js +10 -0
- package/dist/parsers/shapefile/dbf.d.ts +23 -0
- package/dist/parsers/shapefile/dbf.js +112 -0
- package/dist/parsers/shapefile/index.d.ts +44 -0
- package/dist/parsers/shapefile/index.js +119 -0
- package/dist/parsers/shapefile/reproject.d.ts +10 -0
- package/dist/parsers/shapefile/reproject.js +41 -0
- package/dist/parsers/slide-model.d.ts +106 -0
- package/dist/parsers/slide-model.js +1 -0
- package/dist/parsers/toml/index.d.ts +15 -0
- package/dist/parsers/toml/index.js +18 -0
- package/dist/parsers/toml/model.d.ts +15 -0
- package/dist/parsers/toml/model.js +1 -0
- package/dist/parsers/toml/parse.d.ts +16 -0
- package/dist/parsers/toml/parse.js +266 -0
- package/dist/parsers/types.d.ts +103 -0
- package/dist/parsers/types.js +106 -0
- package/dist/parsers/video/index.d.ts +6 -0
- package/dist/parsers/video/index.js +2 -0
- package/dist/parsers/yaml/index.d.ts +16 -0
- package/dist/parsers/yaml/index.js +58 -0
- package/dist/parsers/yaml/model.d.ts +22 -0
- package/dist/parsers/yaml/model.js +1 -0
- package/dist/parsers/yaml/self-loading.d.ts +3 -0
- package/dist/parsers/yaml/self-loading.js +19 -0
- package/dist/parsers/zip-scan.d.ts +8 -0
- package/dist/parsers/zip-scan.js +80 -0
- package/dist/registry/container.d.ts +17 -0
- package/dist/registry/container.js +49 -0
- package/dist/registry/index.d.ts +133 -0
- package/dist/registry/index.js +348 -0
- package/dist/registry/probe.d.ts +3 -0
- package/dist/registry/probe.js +177 -0
- package/dist/registry/sniff.d.ts +17 -0
- package/dist/registry/sniff.js +57 -0
- package/dist/styles/archive.css +10 -0
- package/dist/styles/audio.css +66 -0
- package/dist/styles/automotive.css +1 -0
- package/dist/styles/csv.css +262 -0
- package/dist/styles/excel.css +212 -0
- package/dist/styles/fallback.css +48 -0
- package/dist/styles/hdf5.css +3 -0
- package/dist/styles/hwp.css +12 -0
- package/dist/styles/image.css +115 -0
- package/dist/styles/json.css +342 -0
- package/dist/styles/jsonl.css +1 -0
- package/dist/styles/markdown.css +10 -0
- package/dist/styles/mat.css +2 -0
- package/dist/styles/mermaid.css +1 -0
- package/dist/styles/parquet.css +2 -0
- package/dist/styles/pdf.css +551 -0
- package/dist/styles/plantuml.css +1 -0
- package/dist/styles/ppt.css +1 -0
- package/dist/styles/proto.css +2 -0
- package/dist/styles/psd.css +1 -0
- package/dist/styles/shapefile.css +25 -0
- package/dist/styles/toml.css +1 -0
- package/dist/styles/video.css +74 -0
- package/dist/styles/word.css +685 -0
- package/dist/styles/yaml.css +1 -0
- package/dist/viewers/a2l/index.d.ts +15 -0
- package/dist/viewers/a2l/index.js +5 -0
- package/dist/viewers/archive/archive-preview-decoder.d.ts +4 -0
- package/dist/viewers/archive/archive-preview-decoder.js +223 -0
- package/dist/viewers/archive/controller.d.ts +24 -0
- package/dist/viewers/archive/controller.js +18 -0
- package/dist/viewers/archive/index.d.ts +25 -0
- package/dist/viewers/archive/index.js +230 -0
- package/dist/viewers/archive/preview.d.ts +2 -0
- package/dist/viewers/archive/preview.js +2 -0
- package/dist/viewers/archive/styles.d.ts +1 -0
- package/dist/viewers/archive/styles.js +11 -0
- package/dist/viewers/arxml/index.d.ts +15 -0
- package/dist/viewers/arxml/index.js +5 -0
- package/dist/viewers/asc/index.d.ts +15 -0
- package/dist/viewers/asc/index.js +5 -0
- package/dist/viewers/audio/controller.d.ts +33 -0
- package/dist/viewers/audio/controller.js +49 -0
- package/dist/viewers/audio/engine.d.ts +44 -0
- package/dist/viewers/audio/engine.js +108 -0
- package/dist/viewers/audio/index.d.ts +92 -0
- package/dist/viewers/audio/index.js +448 -0
- package/dist/viewers/audio/self-loading.d.ts +7 -0
- package/dist/viewers/audio/self-loading.js +42 -0
- package/dist/viewers/audio/styles.d.ts +1 -0
- package/dist/viewers/audio/styles.js +73 -0
- package/dist/viewers/automotive/index.d.ts +8 -0
- package/dist/viewers/automotive/index.js +152 -0
- package/dist/viewers/automotive/styles.d.ts +1 -0
- package/dist/viewers/automotive/styles.js +1 -0
- package/dist/viewers/avro/index.d.ts +15 -0
- package/dist/viewers/avro/index.js +5 -0
- package/dist/viewers/bag/index.d.ts +15 -0
- package/dist/viewers/bag/index.js +5 -0
- package/dist/viewers/blf/index.d.ts +15 -0
- package/dist/viewers/blf/index.js +5 -0
- package/dist/viewers/csv/controller.d.ts +126 -0
- package/dist/viewers/csv/controller.js +598 -0
- package/dist/viewers/csv/index.d.ts +48 -0
- package/dist/viewers/csv/index.js +846 -0
- package/dist/viewers/csv/styles.d.ts +1 -0
- package/dist/viewers/csv/styles.js +267 -0
- package/dist/viewers/db3/index.d.ts +15 -0
- package/dist/viewers/db3/index.js +5 -0
- package/dist/viewers/dbc/index.d.ts +15 -0
- package/dist/viewers/dbc/index.js +5 -0
- package/dist/viewers/diagram.d.ts +14 -0
- package/dist/viewers/diagram.js +95 -0
- package/dist/viewers/excel/controller.d.ts +108 -0
- package/dist/viewers/excel/controller.js +548 -0
- package/dist/viewers/excel/grid.d.ts +28 -0
- package/dist/viewers/excel/grid.js +99 -0
- package/dist/viewers/excel/index.d.ts +37 -0
- package/dist/viewers/excel/index.js +620 -0
- package/dist/viewers/excel/self-loading.d.ts +6 -0
- package/dist/viewers/excel/self-loading.js +13 -0
- package/dist/viewers/excel/styles.d.ts +1 -0
- package/dist/viewers/excel/styles.js +216 -0
- package/dist/viewers/fallback/index.d.ts +17 -0
- package/dist/viewers/fallback/index.js +130 -0
- package/dist/viewers/fallback/styles.d.ts +1 -0
- package/dist/viewers/fallback/styles.js +49 -0
- package/dist/viewers/hdf5/index.d.ts +19 -0
- package/dist/viewers/hdf5/index.js +170 -0
- package/dist/viewers/hdf5/styles.d.ts +1 -0
- package/dist/viewers/hdf5/styles.js +4 -0
- package/dist/viewers/hwp/controller.d.ts +19 -0
- package/dist/viewers/hwp/controller.js +22 -0
- package/dist/viewers/hwp/index.d.ts +39 -0
- package/dist/viewers/hwp/index.js +271 -0
- package/dist/viewers/hwp/self-loading.d.ts +2 -0
- package/dist/viewers/hwp/self-loading.js +6 -0
- package/dist/viewers/hwp/styles.d.ts +1 -0
- package/dist/viewers/hwp/styles.js +13 -0
- package/dist/viewers/image/controller.d.ts +210 -0
- package/dist/viewers/image/controller.js +336 -0
- package/dist/viewers/image/decode.d.ts +54 -0
- package/dist/viewers/image/decode.js +150 -0
- package/dist/viewers/image/index.d.ts +38 -0
- package/dist/viewers/image/index.js +749 -0
- package/dist/viewers/image/limits.d.ts +18 -0
- package/dist/viewers/image/limits.js +20 -0
- package/dist/viewers/image/styles.d.ts +1 -0
- package/dist/viewers/image/styles.js +119 -0
- package/dist/viewers/json/controller.d.ts +89 -0
- package/dist/viewers/json/controller.js +411 -0
- package/dist/viewers/json/index.d.ts +25 -0
- package/dist/viewers/json/index.js +503 -0
- package/dist/viewers/json/styles.d.ts +1 -0
- package/dist/viewers/json/styles.js +346 -0
- package/dist/viewers/json/transforms.d.ts +27 -0
- package/dist/viewers/json/transforms.js +253 -0
- package/dist/viewers/jsonl/controller.d.ts +48 -0
- package/dist/viewers/jsonl/controller.js +136 -0
- package/dist/viewers/jsonl/index.d.ts +22 -0
- package/dist/viewers/jsonl/index.js +184 -0
- package/dist/viewers/jsonl/styles.d.ts +1 -0
- package/dist/viewers/jsonl/styles.js +1 -0
- package/dist/viewers/markdown/controller.d.ts +50 -0
- package/dist/viewers/markdown/controller.js +62 -0
- package/dist/viewers/markdown/index.d.ts +64 -0
- package/dist/viewers/markdown/index.js +425 -0
- package/dist/viewers/markdown/math.d.ts +21 -0
- package/dist/viewers/markdown/math.js +100 -0
- package/dist/viewers/markdown/self-loading.d.ts +5 -0
- package/dist/viewers/markdown/self-loading.js +48 -0
- package/dist/viewers/markdown/styles.d.ts +1 -0
- package/dist/viewers/markdown/styles.js +11 -0
- package/dist/viewers/mat/index.d.ts +19 -0
- package/dist/viewers/mat/index.js +113 -0
- package/dist/viewers/mat/styles.d.ts +1 -0
- package/dist/viewers/mat/styles.js +3 -0
- package/dist/viewers/media.d.ts +9 -0
- package/dist/viewers/media.js +77 -0
- package/dist/viewers/mermaid/index.d.ts +17 -0
- package/dist/viewers/mermaid/index.js +6 -0
- package/dist/viewers/mermaid/self-loading.d.ts +2 -0
- package/dist/viewers/mermaid/self-loading.js +2 -0
- package/dist/viewers/mf4/index.d.ts +15 -0
- package/dist/viewers/mf4/index.js +5 -0
- package/dist/viewers/parquet/controller.d.ts +37 -0
- package/dist/viewers/parquet/controller.js +26 -0
- package/dist/viewers/parquet/index.d.ts +23 -0
- package/dist/viewers/parquet/index.js +134 -0
- package/dist/viewers/parquet/styles.d.ts +1 -0
- package/dist/viewers/parquet/styles.js +3 -0
- package/dist/viewers/pcap/index.d.ts +15 -0
- package/dist/viewers/pcap/index.js +5 -0
- package/dist/viewers/pcapng/index.d.ts +15 -0
- package/dist/viewers/pcapng/index.js +5 -0
- package/dist/viewers/pdf/controller.d.ts +120 -0
- package/dist/viewers/pdf/controller.js +191 -0
- package/dist/viewers/pdf/editing.d.ts +26 -0
- package/dist/viewers/pdf/editing.js +265 -0
- package/dist/viewers/pdf/index.d.ts +88 -0
- package/dist/viewers/pdf/index.js +1400 -0
- package/dist/viewers/pdf/self-loading.d.ts +6 -0
- package/dist/viewers/pdf/self-loading.js +14 -0
- package/dist/viewers/pdf/styles.d.ts +1 -0
- package/dist/viewers/pdf/styles.js +555 -0
- package/dist/viewers/plantuml/index.d.ts +17 -0
- package/dist/viewers/plantuml/index.js +6 -0
- package/dist/viewers/plantuml/self-loading.d.ts +2 -0
- package/dist/viewers/plantuml/self-loading.js +2 -0
- package/dist/viewers/ppt/chart.d.ts +8 -0
- package/dist/viewers/ppt/chart.js +83 -0
- package/dist/viewers/ppt/controller.d.ts +34 -0
- package/dist/viewers/ppt/controller.js +27 -0
- package/dist/viewers/ppt/index.d.ts +33 -0
- package/dist/viewers/ppt/index.js +101 -0
- package/dist/viewers/ppt/render.d.ts +5 -0
- package/dist/viewers/ppt/render.js +144 -0
- package/dist/viewers/ppt/self-loading.d.ts +9 -0
- package/dist/viewers/ppt/self-loading.js +4 -0
- package/dist/viewers/ppt/styles.d.ts +1 -0
- package/dist/viewers/ppt/styles.js +1 -0
- package/dist/viewers/proto/index.d.ts +17 -0
- package/dist/viewers/proto/index.js +170 -0
- package/dist/viewers/proto/styles.d.ts +1 -0
- package/dist/viewers/proto/styles.js +2 -0
- package/dist/viewers/psd/index.d.ts +27 -0
- package/dist/viewers/psd/index.js +39 -0
- package/dist/viewers/psd/self-loading.d.ts +10 -0
- package/dist/viewers/psd/self-loading.js +20 -0
- package/dist/viewers/psd/styles.d.ts +1 -0
- package/dist/viewers/psd/styles.js +1 -0
- package/dist/viewers/reqif/index.d.ts +15 -0
- package/dist/viewers/reqif/index.js +5 -0
- package/dist/viewers/shapefile/index.d.ts +35 -0
- package/dist/viewers/shapefile/index.js +336 -0
- package/dist/viewers/shapefile/self-loading.d.ts +7 -0
- package/dist/viewers/shapefile/self-loading.js +32 -0
- package/dist/viewers/shapefile/styles.d.ts +1 -0
- package/dist/viewers/shapefile/styles.js +28 -0
- package/dist/viewers/stp/index.d.ts +15 -0
- package/dist/viewers/stp/index.js +5 -0
- package/dist/viewers/structured-styles.d.ts +1 -0
- package/dist/viewers/structured-styles.js +1 -0
- package/dist/viewers/structured.d.ts +36 -0
- package/dist/viewers/structured.js +138 -0
- package/dist/viewers/toml/controller.d.ts +38 -0
- package/dist/viewers/toml/controller.js +61 -0
- package/dist/viewers/toml/index.d.ts +18 -0
- package/dist/viewers/toml/index.js +34 -0
- package/dist/viewers/types.d.ts +36 -0
- package/dist/viewers/types.js +9 -0
- package/dist/viewers/video/controller.d.ts +29 -0
- package/dist/viewers/video/controller.js +54 -0
- package/dist/viewers/video/index.d.ts +18 -0
- package/dist/viewers/video/index.js +305 -0
- package/dist/viewers/video/styles.d.ts +1 -0
- package/dist/viewers/video/styles.js +79 -0
- package/dist/viewers/word/controller.d.ts +19 -0
- package/dist/viewers/word/controller.js +22 -0
- package/dist/viewers/word/docx-preprocess.d.ts +63 -0
- package/dist/viewers/word/docx-preprocess.js +136 -0
- package/dist/viewers/word/index.d.ts +32 -0
- package/dist/viewers/word/index.js +187 -0
- package/dist/viewers/word/normalize-docx.d.ts +2 -0
- package/dist/viewers/word/normalize-docx.js +50 -0
- package/dist/viewers/word/paginate.d.ts +1 -0
- package/dist/viewers/word/paginate.js +1035 -0
- package/dist/viewers/word/self-loading.d.ts +2 -0
- package/dist/viewers/word/self-loading.js +7 -0
- package/dist/viewers/word/styles.d.ts +1 -0
- package/dist/viewers/word/styles.js +686 -0
- package/dist/viewers/yaml/controller.d.ts +42 -0
- package/dist/viewers/yaml/controller.js +41 -0
- package/dist/viewers/yaml/index.d.ts +21 -0
- package/dist/viewers/yaml/index.js +20 -0
- package/package.json +375 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// Recursive-descent JSON parser producing the fidelity-preserving JsonNode
|
|
2
|
+
// tree (docs/parsers/json.md). Separate from the tokenizer (which serves
|
|
3
|
+
// highlighting): this builds the document model with source spans, raw number
|
|
4
|
+
// text, and ordered duplicate keys. Never throws — syntax errors record a
|
|
5
|
+
// diagnostic and stop, yielding whatever prefix parsed (partial recovery).
|
|
6
|
+
import { appendPath, decodeJsonString } from './model.js';
|
|
7
|
+
import { isDigit, isWhitespace, scanNumber, scanString } from './tokenizer.js';
|
|
8
|
+
export const JSON_DEFAULT_MAX_DEPTH = 500;
|
|
9
|
+
export function parseJsonDocument(text, options = {}) {
|
|
10
|
+
const maxDepth = options.maxDepth ?? JSON_DEFAULT_MAX_DEPTH;
|
|
11
|
+
const onNode = options.onNode;
|
|
12
|
+
const len = text.length;
|
|
13
|
+
const diagnostics = [];
|
|
14
|
+
let pos = 0;
|
|
15
|
+
let stopped = false;
|
|
16
|
+
let stopReason = null;
|
|
17
|
+
let errored = false;
|
|
18
|
+
let trailing = false;
|
|
19
|
+
const skipWs = () => {
|
|
20
|
+
while (pos < len && isWhitespace(text.charCodeAt(pos)))
|
|
21
|
+
pos++;
|
|
22
|
+
};
|
|
23
|
+
const fail = (code, at) => {
|
|
24
|
+
if (errored)
|
|
25
|
+
return; // first error only — the rest is unreliable
|
|
26
|
+
errored = true;
|
|
27
|
+
diagnostics.push({
|
|
28
|
+
severity: 'error',
|
|
29
|
+
code,
|
|
30
|
+
messageKey: `diag.${code}`,
|
|
31
|
+
location: `@${at}`
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
/** Register a node with the checkpoint; returns false if parsing must stop. */
|
|
35
|
+
const enter = () => {
|
|
36
|
+
if (!onNode)
|
|
37
|
+
return true;
|
|
38
|
+
const reason = onNode();
|
|
39
|
+
if (reason) {
|
|
40
|
+
stopped = true;
|
|
41
|
+
stopReason = reason;
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
return true;
|
|
45
|
+
};
|
|
46
|
+
function parseValue(key, path, depth) {
|
|
47
|
+
if (stopped || errored)
|
|
48
|
+
return null;
|
|
49
|
+
if (!enter())
|
|
50
|
+
return null;
|
|
51
|
+
if (depth > maxDepth) {
|
|
52
|
+
diagnostics.push({
|
|
53
|
+
severity: 'warning',
|
|
54
|
+
code: 'json.max-depth',
|
|
55
|
+
messageKey: 'diag.json.max-depth',
|
|
56
|
+
location: path
|
|
57
|
+
});
|
|
58
|
+
stopped = true;
|
|
59
|
+
stopReason = 'limit';
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
skipWs();
|
|
63
|
+
if (pos >= len) {
|
|
64
|
+
fail('json.unexpected-eof', pos);
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
const ch = text.charCodeAt(pos);
|
|
68
|
+
if (ch === 0x7b /* { */)
|
|
69
|
+
return parseObject(key, path, depth);
|
|
70
|
+
if (ch === 0x5b /* [ */)
|
|
71
|
+
return parseArray(key, path, depth);
|
|
72
|
+
if (ch === 0x22 /* " */)
|
|
73
|
+
return parseString(key, path, depth);
|
|
74
|
+
if (ch === 0x2d /* - */ || isDigit(ch))
|
|
75
|
+
return parseNumber(key, path, depth);
|
|
76
|
+
if (text.startsWith('true', pos))
|
|
77
|
+
return literal(key, path, depth, 'boolean', true, 4);
|
|
78
|
+
if (text.startsWith('false', pos))
|
|
79
|
+
return literal(key, path, depth, 'boolean', false, 5);
|
|
80
|
+
if (text.startsWith('null', pos))
|
|
81
|
+
return literal(key, path, depth, 'null', null, 4);
|
|
82
|
+
fail('json.invalid-token', pos);
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
function literal(key, path, depth, kind, value, length) {
|
|
86
|
+
const start = pos;
|
|
87
|
+
pos += length;
|
|
88
|
+
return { kind, key, path, depth, value, span: { start, end: pos } };
|
|
89
|
+
}
|
|
90
|
+
function parseString(key, path, depth) {
|
|
91
|
+
const start = pos;
|
|
92
|
+
const scan = scanString(text, pos);
|
|
93
|
+
const raw = text.slice(pos, pos + scan.length);
|
|
94
|
+
pos += scan.length;
|
|
95
|
+
if (!scan.terminated) {
|
|
96
|
+
fail('json.unterminated-string', start);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
const err = stringContentError(raw);
|
|
100
|
+
if (err)
|
|
101
|
+
fail(err, start);
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
kind: 'string',
|
|
105
|
+
key,
|
|
106
|
+
path,
|
|
107
|
+
depth,
|
|
108
|
+
value: decodeJsonString(raw),
|
|
109
|
+
span: { start, end: pos }
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
function parseNumber(key, path, depth) {
|
|
113
|
+
const start = pos;
|
|
114
|
+
const n = scanNumber(text, pos);
|
|
115
|
+
if (n === 0) {
|
|
116
|
+
fail('json.invalid-token', start);
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
pos += n;
|
|
120
|
+
return {
|
|
121
|
+
kind: 'number',
|
|
122
|
+
key,
|
|
123
|
+
path,
|
|
124
|
+
depth,
|
|
125
|
+
rawNumber: text.slice(start, pos),
|
|
126
|
+
span: { start, end: pos }
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function parseObject(key, path, depth) {
|
|
130
|
+
const start = pos;
|
|
131
|
+
pos++; // consume '{'
|
|
132
|
+
const children = [];
|
|
133
|
+
const seen = new Set();
|
|
134
|
+
skipWs();
|
|
135
|
+
if (pos < len && text.charCodeAt(pos) === 0x7d /* } */) {
|
|
136
|
+
pos++;
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
for (;;) {
|
|
140
|
+
skipWs();
|
|
141
|
+
if (pos >= len || text.charCodeAt(pos) !== 0x22 /* " */) {
|
|
142
|
+
fail('json.invalid-token', pos);
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
const keyScan = scanString(text, pos);
|
|
146
|
+
const keyRaw = text.slice(pos, pos + keyScan.length);
|
|
147
|
+
const keyStr = decodeJsonString(keyRaw);
|
|
148
|
+
if (!keyScan.terminated) {
|
|
149
|
+
fail('json.unterminated-string', pos);
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
const keyErr = stringContentError(keyRaw);
|
|
153
|
+
if (keyErr) {
|
|
154
|
+
fail(keyErr, pos);
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
pos += keyScan.length;
|
|
158
|
+
skipWs();
|
|
159
|
+
if (pos >= len || text.charCodeAt(pos) !== 0x3a /* : */) {
|
|
160
|
+
fail('json.invalid-token', pos);
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
pos++; // consume ':'
|
|
164
|
+
const childPath = appendPath(path, keyStr, false);
|
|
165
|
+
const child = parseValue(keyStr, childPath, depth + 1);
|
|
166
|
+
if (!child)
|
|
167
|
+
break;
|
|
168
|
+
if (seen.has(keyStr)) {
|
|
169
|
+
diagnostics.push({
|
|
170
|
+
severity: 'warning',
|
|
171
|
+
code: 'json.duplicate-key',
|
|
172
|
+
messageKey: 'diag.json.duplicate-key',
|
|
173
|
+
args: { key: keyStr },
|
|
174
|
+
location: childPath
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
seen.add(keyStr);
|
|
179
|
+
}
|
|
180
|
+
children.push(child);
|
|
181
|
+
skipWs();
|
|
182
|
+
if (pos < len && text.charCodeAt(pos) === 0x2c /* , */) {
|
|
183
|
+
pos++;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (pos < len && text.charCodeAt(pos) === 0x7d /* } */) {
|
|
187
|
+
pos++;
|
|
188
|
+
break;
|
|
189
|
+
}
|
|
190
|
+
fail('json.invalid-token', pos);
|
|
191
|
+
break;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return { kind: 'object', key, path, depth, children, span: { start, end: pos } };
|
|
195
|
+
}
|
|
196
|
+
function parseArray(key, path, depth) {
|
|
197
|
+
const start = pos;
|
|
198
|
+
pos++; // consume '['
|
|
199
|
+
const children = [];
|
|
200
|
+
skipWs();
|
|
201
|
+
if (pos < len && text.charCodeAt(pos) === 0x5d /* ] */) {
|
|
202
|
+
pos++;
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
let index = 0;
|
|
206
|
+
for (;;) {
|
|
207
|
+
const childPath = appendPath(path, String(index), true);
|
|
208
|
+
const child = parseValue(String(index), childPath, depth + 1);
|
|
209
|
+
if (!child)
|
|
210
|
+
break;
|
|
211
|
+
children.push(child);
|
|
212
|
+
index++;
|
|
213
|
+
skipWs();
|
|
214
|
+
if (pos < len && text.charCodeAt(pos) === 0x2c /* , */) {
|
|
215
|
+
pos++;
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
if (pos < len && text.charCodeAt(pos) === 0x5d /* ] */) {
|
|
219
|
+
pos++;
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
fail('json.invalid-token', pos);
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return { kind: 'array', key, path, depth, children, span: { start, end: pos } };
|
|
227
|
+
}
|
|
228
|
+
skipWs();
|
|
229
|
+
const root = pos >= len ? (fail('json.unexpected-eof', pos), null) : parseValue('$', '$', 0);
|
|
230
|
+
if (root && !stopped && !errored) {
|
|
231
|
+
skipWs();
|
|
232
|
+
if (pos < len) {
|
|
233
|
+
trailing = true;
|
|
234
|
+
diagnostics.push({
|
|
235
|
+
severity: 'warning',
|
|
236
|
+
code: 'json.trailing-content',
|
|
237
|
+
messageKey: 'diag.json.trailing-content',
|
|
238
|
+
location: `@${pos}`
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return { root, diagnostics, stopped, stopReason, errored, trailing };
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Validate a terminated JSON string literal's content per RFC 8259: no
|
|
246
|
+
* unescaped control characters (< U+0020), only the allowed escapes
|
|
247
|
+
* (`" \ / b f n r t`), and `\u` followed by exactly four hex digits. Returns a
|
|
248
|
+
* diagnostic code or null. This is what keeps invalid strings ("\q", "\u12G4",
|
|
249
|
+
* a raw newline) from being reported as valid JSON (P1a).
|
|
250
|
+
*/
|
|
251
|
+
function stringContentError(raw) {
|
|
252
|
+
const end = raw.length - 1; // exclude the closing quote
|
|
253
|
+
let i = 1; // skip the opening quote
|
|
254
|
+
while (i < end) {
|
|
255
|
+
const c = raw.charCodeAt(i);
|
|
256
|
+
if (c === 0x5c /* \ */) {
|
|
257
|
+
const next = raw[i + 1];
|
|
258
|
+
if (next === undefined)
|
|
259
|
+
return 'json.invalid-escape';
|
|
260
|
+
if (next === '"' || next === '\\' || next === '/' || next === 'b' ||
|
|
261
|
+
next === 'f' || next === 'n' || next === 'r' || next === 't') {
|
|
262
|
+
i += 2;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (next === 'u') {
|
|
266
|
+
const hex = raw.slice(i + 2, i + 6);
|
|
267
|
+
if (!/^[0-9a-fA-F]{4}$/.test(hex))
|
|
268
|
+
return 'json.invalid-unicode-escape';
|
|
269
|
+
i += 6;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
return 'json.invalid-escape';
|
|
273
|
+
}
|
|
274
|
+
if (c < 0x20)
|
|
275
|
+
return 'json.control-char';
|
|
276
|
+
i++;
|
|
277
|
+
}
|
|
278
|
+
return null;
|
|
279
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type JsonTokenKind = 'key' | 'string' | 'number' | 'bool' | 'null' | 'punct' | 'whitespace' | 'unknown';
|
|
2
|
+
export interface JsonToken {
|
|
3
|
+
kind: JsonTokenKind;
|
|
4
|
+
text: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function tokenizeJson(input: string): JsonToken[];
|
|
7
|
+
interface StringScan {
|
|
8
|
+
length: number;
|
|
9
|
+
terminated: boolean;
|
|
10
|
+
}
|
|
11
|
+
/** Scan a string literal at `start` (points at the opening `"`). Honours `\`. */
|
|
12
|
+
export declare function scanString(input: string, start: number): StringScan;
|
|
13
|
+
/** Scan a JSON number at `start`; returns chars consumed, or 0 if not a number. */
|
|
14
|
+
export declare function scanNumber(input: string, start: number): number;
|
|
15
|
+
export declare function isWhitespace(ch: number): boolean;
|
|
16
|
+
export declare function isDigit(ch: number): boolean;
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// Pure JSON tokenizer — flat token list for source syntax highlighting
|
|
2
|
+
// (docs/viewers/json.md J14 verbatim view). Ported from the chrome viewer
|
|
3
|
+
// (jsonTokenizer). DOM-free and never throws: malformed input falls through to
|
|
4
|
+
// `unknown` tokens so the caller can still render the prefix that scanned
|
|
5
|
+
// cleanly. The sum of token.text equals the input, so the source can be
|
|
6
|
+
// reconstructed by concatenating tokens in order.
|
|
7
|
+
//
|
|
8
|
+
// The document model (parse.ts) is a separate recursive-descent parser; this
|
|
9
|
+
// tokenizer only serves highlighting, mirroring the chrome split.
|
|
10
|
+
export function tokenizeJson(input) {
|
|
11
|
+
const tokens = [];
|
|
12
|
+
if (typeof input !== 'string' || input.length === 0) {
|
|
13
|
+
return tokens;
|
|
14
|
+
}
|
|
15
|
+
const len = input.length;
|
|
16
|
+
let i = 0;
|
|
17
|
+
while (i < len) {
|
|
18
|
+
const ch = input.charCodeAt(i);
|
|
19
|
+
if (isWhitespace(ch)) {
|
|
20
|
+
const start = i;
|
|
21
|
+
i++;
|
|
22
|
+
while (i < len && isWhitespace(input.charCodeAt(i)))
|
|
23
|
+
i++;
|
|
24
|
+
tokens.push({ kind: 'whitespace', text: input.slice(start, i) });
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
if (ch === 0x7b /* { */ ||
|
|
28
|
+
ch === 0x7d /* } */ ||
|
|
29
|
+
ch === 0x5b /* [ */ ||
|
|
30
|
+
ch === 0x5d /* ] */ ||
|
|
31
|
+
ch === 0x2c /* , */ ||
|
|
32
|
+
ch === 0x3a /* : */) {
|
|
33
|
+
tokens.push({ kind: 'punct', text: input[i] });
|
|
34
|
+
i++;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (ch === 0x22 /* " */) {
|
|
38
|
+
const consumed = scanString(input, i);
|
|
39
|
+
const text = input.slice(i, i + consumed.length);
|
|
40
|
+
i += consumed.length;
|
|
41
|
+
if (consumed.terminated) {
|
|
42
|
+
let j = i;
|
|
43
|
+
while (j < len && isWhitespace(input.charCodeAt(j)))
|
|
44
|
+
j++;
|
|
45
|
+
const isKey = j < len && input.charCodeAt(j) === 0x3a; /* : */
|
|
46
|
+
tokens.push({ kind: isKey ? 'key' : 'string', text });
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
tokens.push({ kind: 'unknown', text });
|
|
50
|
+
}
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (ch === 0x74 /* t */ && input.startsWith('true', i)) {
|
|
54
|
+
tokens.push({ kind: 'bool', text: 'true' });
|
|
55
|
+
i += 4;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (ch === 0x66 /* f */ && input.startsWith('false', i)) {
|
|
59
|
+
tokens.push({ kind: 'bool', text: 'false' });
|
|
60
|
+
i += 5;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (ch === 0x6e /* n */ && input.startsWith('null', i)) {
|
|
64
|
+
tokens.push({ kind: 'null', text: 'null' });
|
|
65
|
+
i += 4;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (ch === 0x2d /* - */ || isDigit(ch)) {
|
|
69
|
+
const consumed = scanNumber(input, i);
|
|
70
|
+
if (consumed > 0) {
|
|
71
|
+
tokens.push({ kind: 'number', text: input.slice(i, i + consumed) });
|
|
72
|
+
i += consumed;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const start = i;
|
|
77
|
+
i++;
|
|
78
|
+
while (i < len && !canStartToken(input.charCodeAt(i)))
|
|
79
|
+
i++;
|
|
80
|
+
tokens.push({ kind: 'unknown', text: input.slice(start, i) });
|
|
81
|
+
}
|
|
82
|
+
return tokens;
|
|
83
|
+
}
|
|
84
|
+
/** Scan a string literal at `start` (points at the opening `"`). Honours `\`. */
|
|
85
|
+
export function scanString(input, start) {
|
|
86
|
+
const len = input.length;
|
|
87
|
+
let i = start + 1;
|
|
88
|
+
while (i < len) {
|
|
89
|
+
const c = input.charCodeAt(i);
|
|
90
|
+
if (c === 0x5c /* \ */) {
|
|
91
|
+
i += 2;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (c === 0x22 /* " */) {
|
|
95
|
+
return { length: i - start + 1, terminated: true };
|
|
96
|
+
}
|
|
97
|
+
i++;
|
|
98
|
+
}
|
|
99
|
+
return { length: len - start, terminated: false };
|
|
100
|
+
}
|
|
101
|
+
/** Scan a JSON number at `start`; returns chars consumed, or 0 if not a number. */
|
|
102
|
+
export function scanNumber(input, start) {
|
|
103
|
+
const len = input.length;
|
|
104
|
+
let i = start;
|
|
105
|
+
if (i < len && input.charCodeAt(i) === 0x2d /* - */)
|
|
106
|
+
i++;
|
|
107
|
+
if (i >= len)
|
|
108
|
+
return 0;
|
|
109
|
+
const c = input.charCodeAt(i);
|
|
110
|
+
if (c === 0x30 /* 0 */) {
|
|
111
|
+
i++;
|
|
112
|
+
}
|
|
113
|
+
else if (c >= 0x31 /* 1 */ && c <= 0x39 /* 9 */) {
|
|
114
|
+
i++;
|
|
115
|
+
while (i < len && isDigit(input.charCodeAt(i)))
|
|
116
|
+
i++;
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
return 0;
|
|
120
|
+
}
|
|
121
|
+
if (i < len && input.charCodeAt(i) === 0x2e /* . */) {
|
|
122
|
+
i++;
|
|
123
|
+
const fracStart = i;
|
|
124
|
+
while (i < len && isDigit(input.charCodeAt(i)))
|
|
125
|
+
i++;
|
|
126
|
+
if (i === fracStart)
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
if (i < len) {
|
|
130
|
+
const e = input.charCodeAt(i);
|
|
131
|
+
if (e === 0x65 /* e */ || e === 0x45 /* E */) {
|
|
132
|
+
i++;
|
|
133
|
+
if (i < len) {
|
|
134
|
+
const sign = input.charCodeAt(i);
|
|
135
|
+
if (sign === 0x2b /* + */ || sign === 0x2d /* - */)
|
|
136
|
+
i++;
|
|
137
|
+
}
|
|
138
|
+
const expStart = i;
|
|
139
|
+
while (i < len && isDigit(input.charCodeAt(i)))
|
|
140
|
+
i++;
|
|
141
|
+
if (i === expStart)
|
|
142
|
+
return 0;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return i - start;
|
|
146
|
+
}
|
|
147
|
+
export function isWhitespace(ch) {
|
|
148
|
+
return (ch === 0x20 ||
|
|
149
|
+
ch === 0x09 ||
|
|
150
|
+
ch === 0x0a ||
|
|
151
|
+
ch === 0x0d ||
|
|
152
|
+
ch === 0x0b ||
|
|
153
|
+
ch === 0x0c);
|
|
154
|
+
}
|
|
155
|
+
export function isDigit(ch) {
|
|
156
|
+
return ch >= 0x30 && ch <= 0x39;
|
|
157
|
+
}
|
|
158
|
+
function canStartToken(ch) {
|
|
159
|
+
if (isWhitespace(ch))
|
|
160
|
+
return true;
|
|
161
|
+
if (isDigit(ch))
|
|
162
|
+
return true;
|
|
163
|
+
return (ch === 0x7b || // {
|
|
164
|
+
ch === 0x7d || // }
|
|
165
|
+
ch === 0x5b || // [
|
|
166
|
+
ch === 0x5d || // ]
|
|
167
|
+
ch === 0x2c || // ,
|
|
168
|
+
ch === 0x3a || // :
|
|
169
|
+
ch === 0x22 || // "
|
|
170
|
+
ch === 0x2d || // -
|
|
171
|
+
ch === 0x74 || // t
|
|
172
|
+
ch === 0x66 || // f
|
|
173
|
+
ch === 0x6e // n
|
|
174
|
+
);
|
|
175
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ParseOutcome } from '../types.js';
|
|
2
|
+
import type { JsonlDocument } from './model.js';
|
|
3
|
+
export * from './model.js';
|
|
4
|
+
export { parseJsonlText } from './parse.js';
|
|
5
|
+
export declare const JSONL_INPUT_OWNERSHIP: "borrows";
|
|
6
|
+
export interface JsonlParseOptions {
|
|
7
|
+
limits?: {
|
|
8
|
+
maxInputBytes?: number;
|
|
9
|
+
maxEntries?: number;
|
|
10
|
+
maxLineBytes?: number;
|
|
11
|
+
};
|
|
12
|
+
signal?: AbortSignal;
|
|
13
|
+
}
|
|
14
|
+
export declare function parseJsonl(input: Uint8Array | string, options?: JsonlParseOptions): ParseOutcome<JsonlDocument>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { decodeUtf8, utf8ByteLength } from '../types.js';
|
|
2
|
+
import { parseJsonlText } from './parse.js';
|
|
3
|
+
export * from './model.js';
|
|
4
|
+
export { parseJsonlText } from './parse.js';
|
|
5
|
+
export const JSONL_INPUT_OWNERSHIP = 'borrows';
|
|
6
|
+
export function parseJsonl(input, options = {}) {
|
|
7
|
+
const started = Date.now(), max = options.limits?.maxInputBytes ?? 256 * 1024 * 1024, bytes = typeof input === 'string' ? utf8ByteLength(input) : input.byteLength;
|
|
8
|
+
const finish = (result) => ({ result, execution: { workerUsed: false, hardLimitEnforced: false, elapsedMillis: Date.now() - started } });
|
|
9
|
+
if (bytes > max)
|
|
10
|
+
return finish({ status: 'failed', failure: { code: 'limit-exceeded', retryable: false, messageKey: 'diag.limit-exceeded.input', args: { maxBytes: max } }, diagnostics: [] });
|
|
11
|
+
if (options.signal?.aborted)
|
|
12
|
+
return finish({ status: 'failed', failure: { code: 'aborted', retryable: true, messageKey: 'diag.aborted' }, diagnostics: [] });
|
|
13
|
+
const parsed = parseJsonlText(typeof input === 'string' ? input : decodeUtf8(input), options.limits?.maxEntries ?? 1_000_000);
|
|
14
|
+
const long = parsed.document.entries.find(e => utf8ByteLength(e.raw) > (options.limits?.maxLineBytes ?? 1024 * 1024));
|
|
15
|
+
if (long)
|
|
16
|
+
parsed.diagnostics.push({ severity: 'warning', code: 'jsonl.line-limit', messageKey: 'diag.jsonl.line-limit', args: { line: long.line }, location: `line:${long.line}` });
|
|
17
|
+
return finish(parsed.limited || parsed.diagnostics.length ? { status: 'partial', document: parsed.document, diagnostics: parsed.diagnostics } : { status: 'ok', document: parsed.document, diagnostics: [] });
|
|
18
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { JsonNode } from '../json/index.js';
|
|
2
|
+
export interface JsonlEntry {
|
|
3
|
+
/** Immutable controller identity; physical `line` is presentation only. */
|
|
4
|
+
id: string;
|
|
5
|
+
line: number;
|
|
6
|
+
raw: string;
|
|
7
|
+
lineEnding: '\n' | '\r\n' | '';
|
|
8
|
+
value: JsonNode | null;
|
|
9
|
+
diagnostics: import('../types.js').Diagnostic[];
|
|
10
|
+
}
|
|
11
|
+
export interface JsonlDocument {
|
|
12
|
+
text: string;
|
|
13
|
+
entries: JsonlEntry[];
|
|
14
|
+
}
|
|
15
|
+
/** Deterministic append-only preview protocol (docs/viewers/jsonl.md JNL3). */
|
|
16
|
+
export interface JsonlPagedInput {
|
|
17
|
+
initialData: Uint8Array;
|
|
18
|
+
totalBytes?: number;
|
|
19
|
+
loadMore?: () => Promise<{
|
|
20
|
+
data: Uint8Array;
|
|
21
|
+
done: boolean;
|
|
22
|
+
}>;
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { parseJsonDocument } from '../json/index.js';
|
|
2
|
+
export function parseJsonlText(text, maxEntries) {
|
|
3
|
+
const entries = [];
|
|
4
|
+
const diagnostics = [];
|
|
5
|
+
const lines = text.match(/.*(?:\r\n|\n|$)/g) ?? [];
|
|
6
|
+
let line = 1;
|
|
7
|
+
let limited = false;
|
|
8
|
+
for (const item of lines) {
|
|
9
|
+
if (!item && line > 1)
|
|
10
|
+
continue;
|
|
11
|
+
if (entries.length >= maxEntries) {
|
|
12
|
+
limited = true;
|
|
13
|
+
break;
|
|
14
|
+
}
|
|
15
|
+
const ending = item.endsWith('\r\n') ? '\r\n' : item.endsWith('\n') ? '\n' : '';
|
|
16
|
+
const raw = ending ? item.slice(0, -ending.length) : item;
|
|
17
|
+
if (!raw.trim()) {
|
|
18
|
+
entries.push({ id: `line:${line}`, line, raw, lineEnding: ending, value: null, diagnostics: [] });
|
|
19
|
+
line++;
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
const parsed = parseJsonDocument(raw);
|
|
23
|
+
const entryDiagnostics = parsed.diagnostics;
|
|
24
|
+
if (!parsed.root || parsed.errored || parsed.trailing) {
|
|
25
|
+
const diagnostic = { severity: 'warning', code: 'invalid-jsonl-line', messageKey: 'diag.jsonl.invalid-line', args: { line }, location: `line:${line}` };
|
|
26
|
+
entries.push({ id: `line:${line}`, line, raw, lineEnding: ending, value: parsed.root, diagnostics: [...entryDiagnostics, diagnostic] });
|
|
27
|
+
diagnostics.push(diagnostic);
|
|
28
|
+
}
|
|
29
|
+
else
|
|
30
|
+
entries.push({ id: `line:${line}`, line, raw, lineEnding: ending, value: parsed.root, diagnostics: entryDiagnostics });
|
|
31
|
+
line++;
|
|
32
|
+
}
|
|
33
|
+
return { document: { text, entries }, diagnostics, limited };
|
|
34
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type ParseOutcome, type ParseOptions } from '../types.js';
|
|
2
|
+
export interface MarkdownHeading {
|
|
3
|
+
level: number;
|
|
4
|
+
text: string;
|
|
5
|
+
id: string;
|
|
6
|
+
start: number;
|
|
7
|
+
end: number;
|
|
8
|
+
}
|
|
9
|
+
export interface MarkdownBlock {
|
|
10
|
+
start: number;
|
|
11
|
+
end: number;
|
|
12
|
+
}
|
|
13
|
+
export interface MarkdownDocument {
|
|
14
|
+
/** Complete decoded source, retained for source view/edit/writeback. */
|
|
15
|
+
text: string;
|
|
16
|
+
/** Prefix that passed all structural limits and is safe to hand to a renderer. */
|
|
17
|
+
renderText: string;
|
|
18
|
+
headings: MarkdownHeading[];
|
|
19
|
+
blocks: MarkdownBlock[];
|
|
20
|
+
}
|
|
21
|
+
export declare const MARKDOWN_DEFAULT_LIMITS: {
|
|
22
|
+
readonly maxInputBytes: number;
|
|
23
|
+
readonly maxBlocks: 200000;
|
|
24
|
+
readonly maxHeadings: 10000;
|
|
25
|
+
readonly maxDepth: 64;
|
|
26
|
+
};
|
|
27
|
+
export interface MarkdownStructuralLimits {
|
|
28
|
+
maxBlocks: number;
|
|
29
|
+
maxHeadings: number;
|
|
30
|
+
maxDepth: number;
|
|
31
|
+
}
|
|
32
|
+
export interface MarkdownParseOptions extends ParseOptions {
|
|
33
|
+
markdownLimits?: Partial<MarkdownStructuralLimits>;
|
|
34
|
+
}
|
|
35
|
+
/** A deliberately lightweight source index. Rendering is owned by the viewer's
|
|
36
|
+
* marked dependency; this parser gives TOC/search deterministic source spans. */
|
|
37
|
+
export declare function parseMarkdown(data: Uint8Array, options?: MarkdownParseOptions): ParseOutcome<MarkdownDocument>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { decodeUtf8 } from '../types.js';
|
|
2
|
+
export const MARKDOWN_DEFAULT_LIMITS = { maxInputBytes: 10 * 1024 * 1024, maxBlocks: 200_000, maxHeadings: 10_000, maxDepth: 64 };
|
|
3
|
+
/** A deliberately lightweight source index. Rendering is owned by the viewer's
|
|
4
|
+
* marked dependency; this parser gives TOC/search deterministic source spans. */
|
|
5
|
+
export function parseMarkdown(data, options = {}) {
|
|
6
|
+
const started = Date.now();
|
|
7
|
+
if (data.byteLength > (options.limits?.maxInputBytes ?? MARKDOWN_DEFAULT_LIMITS.maxInputBytes)) {
|
|
8
|
+
return { result: { status: 'failed', failure: { code: 'limit-exceeded', retryable: false, messageKey: 'diag.markdown.limit-exceeded' }, diagnostics: [] }, execution: { workerUsed: false, hardLimitEnforced: true, elapsedMillis: Date.now() - started } };
|
|
9
|
+
}
|
|
10
|
+
const maxBlocks = options.markdownLimits?.maxBlocks ?? MARKDOWN_DEFAULT_LIMITS.maxBlocks;
|
|
11
|
+
const maxHeadings = options.markdownLimits?.maxHeadings ?? MARKDOWN_DEFAULT_LIMITS.maxHeadings;
|
|
12
|
+
const maxDepth = options.markdownLimits?.maxDepth ?? MARKDOWN_DEFAULT_LIMITS.maxDepth;
|
|
13
|
+
const text = decodeUtf8(data);
|
|
14
|
+
const headings = [];
|
|
15
|
+
const blocks = [];
|
|
16
|
+
let offset = 0;
|
|
17
|
+
let ordinal = 0;
|
|
18
|
+
for (const line of text.split(/(?<=\n)/)) {
|
|
19
|
+
if (options.signal?.aborted)
|
|
20
|
+
return { result: { status: 'failed', failure: { code: 'aborted', retryable: true, messageKey: 'diag.aborted' }, diagnostics: [] }, execution: { workerUsed: false, hardLimitEnforced: true, elapsedMillis: Date.now() - started } };
|
|
21
|
+
if (blocks.length >= maxBlocks)
|
|
22
|
+
break;
|
|
23
|
+
blocks.push({ start: offset, end: offset + line.length });
|
|
24
|
+
const hit = /^(#{1,6})\s+(.+?)\s*#*\s*(?:\n|$)/.exec(line) ?? /^\s*<h([1-6])[^>]*>(.*?)<\/h\1>\s*$/i.exec(line);
|
|
25
|
+
const depth = markdownDepth(line);
|
|
26
|
+
const headingLevel = hit ? (/^\s*</.test(line) ? Number(hit[1]) : hit[1].length) : 0;
|
|
27
|
+
if ((hit && headingLevel > maxDepth) || depth > maxDepth)
|
|
28
|
+
break;
|
|
29
|
+
if (hit) {
|
|
30
|
+
if (headings.length >= maxHeadings)
|
|
31
|
+
break;
|
|
32
|
+
headings.push({ level: headingLevel, text: hit[2], id: `heading-${ordinal++}`, start: offset, end: offset + line.length });
|
|
33
|
+
}
|
|
34
|
+
offset += line.length;
|
|
35
|
+
}
|
|
36
|
+
const diagnostics = [];
|
|
37
|
+
const partial = offset < text.length;
|
|
38
|
+
if (partial)
|
|
39
|
+
diagnostics.push({ severity: 'warning', code: 'markdown.limit-exceeded', messageKey: 'diag.markdown.limit-exceeded' });
|
|
40
|
+
// `text` must remain complete so a partial preview can never truncate the
|
|
41
|
+
// source on save. Renderers, however, must consume only this bounded prefix.
|
|
42
|
+
const document = { text, renderText: text.slice(0, offset), headings, blocks };
|
|
43
|
+
return { result: partial ? { status: 'partial', document, diagnostics } : { status: 'ok', document, diagnostics }, execution: { workerUsed: false, hardLimitEnforced: true, elapsedMillis: Date.now() - started } };
|
|
44
|
+
}
|
|
45
|
+
function markdownDepth(line) { let i = 0, depth = 0; while (i < line.length) {
|
|
46
|
+
const quote = /^\s*>\s?/.exec(line.slice(i));
|
|
47
|
+
if (quote) {
|
|
48
|
+
depth++;
|
|
49
|
+
i += quote[0].length;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
const list = /^\s*(?:[-+*]|\d+[.)])\s+/.exec(line.slice(i));
|
|
53
|
+
if (list) {
|
|
54
|
+
depth++;
|
|
55
|
+
i += list[0].length;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
break;
|
|
59
|
+
} return depth; }
|