mdld-parse 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +250 -82
- package/dist/crawl.js +660 -0
- package/dist/{index.js → mdld-parse.js} +21 -16
- package/dist/quad-graph.js +1219 -0
- package/package.json +8 -4
package/README.md
CHANGED
|
@@ -38,10 +38,25 @@ Human text → Graph immediately
|
|
|
38
38
|
|
|
39
39
|
## 🚀 Quick Start
|
|
40
40
|
|
|
41
|
+
Install the package in Node environment:
|
|
42
|
+
|
|
41
43
|
```bash
|
|
42
44
|
pnpm install mdld-parse
|
|
43
45
|
```
|
|
44
46
|
|
|
47
|
+
Or use `importmap` in the browser:
|
|
48
|
+
|
|
49
|
+
```html
|
|
50
|
+
<script type="importmap">
|
|
51
|
+
{
|
|
52
|
+
"imports": {
|
|
53
|
+
"mdld-parse": "https://mdld.js.org/mdld-parse.js"
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
</script>
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
|
|
45
60
|
```javascript
|
|
46
61
|
import { parse, generate, merge, render, deconstruct } from 'mdld-parse';
|
|
47
62
|
|
|
@@ -95,6 +110,15 @@ MD-LD removes the intermediary. Writing becomes publishing. Publishing becomes g
|
|
|
95
110
|
- **Text-native agent memory** - LLM Agent memory substrate in plain text — parse context, write knowledge, merge with other agents, all as Markdown files. No database required.
|
|
96
111
|
|
|
97
112
|
|
|
113
|
+
## 📚 Documentation Hub
|
|
114
|
+
|
|
115
|
+
- **📋 [Specification](./spec/index.md)** — Formal specification and test suite
|
|
116
|
+
- **📚 [Grammar](./grammar/index.md)** — EBNF+TextMate grammar specifications
|
|
117
|
+
- **📖 [Documentation](./docs/index.md)** — Complete documentation with guides and references
|
|
118
|
+
- **🎯 [Examples](./examples/index.md)** — Real-world MD-LD examples and use cases
|
|
119
|
+
- **💾 [Reference server](./server/index.md)** - a Node.js implementation of the server for git-powered semantic workflows and publishing
|
|
120
|
+
- **🧩 [Ontologies](./ontologies/index.md)** — W3C and related standard ontologies used in RDF
|
|
121
|
+
|
|
98
122
|
### Real-World Applications
|
|
99
123
|
|
|
100
124
|
#### Personal Knowledge Management
|
|
@@ -516,6 +540,137 @@ const mdld = deconstruct(html);
|
|
|
516
540
|
- Client-side graph extraction: `parse(deconstruct(document.body.innerHTML))`
|
|
517
541
|
- Offline-first apps: cache HTML, reconstruct quads on demand
|
|
518
542
|
|
|
543
|
+
## 🎨 Graph Visualization — `<quad-graph>`
|
|
544
|
+
|
|
545
|
+
Visualize your RDF quads as an interactive force-directed graph with the zero-dependency `<quad-graph>` web component.
|
|
546
|
+
|
|
547
|
+
### Quick Start
|
|
548
|
+
|
|
549
|
+
```html
|
|
550
|
+
<!DOCTYPE html>
|
|
551
|
+
<html>
|
|
552
|
+
<head>
|
|
553
|
+
<script type="importmap">
|
|
554
|
+
{
|
|
555
|
+
"imports": {
|
|
556
|
+
"mdld-parse": "https://mdld.js.org/mdld-parse.js",
|
|
557
|
+
"quad-graph": "https://mdld.js.org/quad-graph.js"
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
</script>
|
|
561
|
+
</head>
|
|
562
|
+
<body>
|
|
563
|
+
<quad-graph height="600"></quad-graph>
|
|
564
|
+
<script type="module">
|
|
565
|
+
import { parse } from 'mdld-parse';
|
|
566
|
+
import 'quad-graph'
|
|
567
|
+
const graph = document.querySelector('quad-graph');
|
|
568
|
+
|
|
569
|
+
const mdld = `[ex] <tag:example.org,2026:>
|
|
570
|
+
# Alice {=ex:alice .prov:Person label}
|
|
571
|
+
[Alice] {ex:name} knows [Bob] {+ex:bob ?ex:knows}`;
|
|
572
|
+
|
|
573
|
+
const result = parse({ text: mdld });
|
|
574
|
+
graph.context = result.context
|
|
575
|
+
graph.quads = result.quads;
|
|
576
|
+
</script>
|
|
577
|
+
</body>
|
|
578
|
+
</html>
|
|
579
|
+
```
|
|
580
|
+
|
|
581
|
+
### Browser Console Usage
|
|
582
|
+
|
|
583
|
+
```javascript
|
|
584
|
+
// Load the modules directly
|
|
585
|
+
const mdld = await import('https://mdld.js.org/mdld-parse.js');
|
|
586
|
+
const graph = await import('https://mdld.js.org/quad-graph.js');
|
|
587
|
+
|
|
588
|
+
// Create and configure the graph
|
|
589
|
+
const quadGraph = document.createElement('quad-graph');
|
|
590
|
+
quadGraph.height = '600';
|
|
591
|
+
document.body.appendChild(quadGraph);
|
|
592
|
+
|
|
593
|
+
// Parse and visualize
|
|
594
|
+
const result = mdld.parse({ text: yourMdldString });
|
|
595
|
+
quadGraph.context = result.context;
|
|
596
|
+
quadGraph.quads = result.quads;
|
|
597
|
+
```
|
|
598
|
+
|
|
599
|
+
### Installation
|
|
600
|
+
|
|
601
|
+
```bash
|
|
602
|
+
pnpm install mdld-parse
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
```javascript
|
|
606
|
+
import { parse } from 'mdld-parse';
|
|
607
|
+
import 'mdld-parse/quad-graph'; // Registers <quad-graph> custom element
|
|
608
|
+
```
|
|
609
|
+
|
|
610
|
+
### Features
|
|
611
|
+
|
|
612
|
+
- **Zero dependencies** — Pure Canvas2D rendering, ~650 LOC
|
|
613
|
+
- **Interactive** — Wheel/pinch zoom, drag background to pan, drag nodes to move, double-click to pin/unpin
|
|
614
|
+
- **Force-directed layout** — Barnes-Hut quadtree for performance, time-sliced across frames
|
|
615
|
+
- **Collision-free labels** — Greedy placement with viewport culling
|
|
616
|
+
- **Dark theme** — High contrast light text on dark background
|
|
617
|
+
- **Customizable** — Attributes for height, charge, link distance, labels
|
|
618
|
+
- **API methods** — `focus(iri)`, `select(iri)`, `fit()`, `pin(iri, on)`, `reheat()`, `getNode(iri)`
|
|
619
|
+
|
|
620
|
+
### Attributes
|
|
621
|
+
|
|
622
|
+
| Attribute | Default | Description |
|
|
623
|
+
|-----------|---------|-------------|
|
|
624
|
+
| `height` | `480px` | Component height |
|
|
625
|
+
| `charge` | `40` | Repulsion force between nodes |
|
|
626
|
+
| `link-dist` | `45` | Preferred link length |
|
|
627
|
+
| `labels` | `on` | Set to `"off"` to hide labels |
|
|
628
|
+
|
|
629
|
+
### API Methods
|
|
630
|
+
|
|
631
|
+
```javascript
|
|
632
|
+
const graph = document.querySelector('quad-graph');
|
|
633
|
+
|
|
634
|
+
// Focus on a specific node
|
|
635
|
+
graph.focus('http://example.org/alice');
|
|
636
|
+
|
|
637
|
+
// Select a node
|
|
638
|
+
graph.select('http://example.org/bob');
|
|
639
|
+
|
|
640
|
+
// Fit graph to viewport
|
|
641
|
+
graph.fit();
|
|
642
|
+
|
|
643
|
+
// Pin/unpin a node
|
|
644
|
+
graph.pin('http://example.org/alice', true);
|
|
645
|
+
|
|
646
|
+
// Reheat the simulation
|
|
647
|
+
graph.reheat();
|
|
648
|
+
|
|
649
|
+
// Get node data
|
|
650
|
+
const nodeData = graph.getNode('http://example.org/alice');
|
|
651
|
+
// { id, label, x, y, degree, pinned, type, literals }
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
### Events
|
|
655
|
+
|
|
656
|
+
```javascript
|
|
657
|
+
graph.addEventListener('node-click', (e) => {
|
|
658
|
+
console.log('Clicked:', e.detail); // IRI string
|
|
659
|
+
});
|
|
660
|
+
|
|
661
|
+
graph.addEventListener('node-select', (e) => {
|
|
662
|
+
console.log('Selected:', e.detail); // IRI string or null
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
graph.addEventListener('node-hover', (e) => {
|
|
666
|
+
console.log('Hovered:', e.detail); // IRI string or null
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
graph.addEventListener('settled', () => {
|
|
670
|
+
console.log('Layout settled');
|
|
671
|
+
});
|
|
672
|
+
```
|
|
673
|
+
|
|
519
674
|
## Styling by Type
|
|
520
675
|
|
|
521
676
|
Use CSS attribute selectors to style elements by their RDF types:
|
|
@@ -529,6 +684,101 @@ Use CSS attribute selectors to style elements by their RDF types:
|
|
|
529
684
|
|
|
530
685
|
/* Style entities with specific IRI */
|
|
531
686
|
[data-iri="http://example.org/alice"] { font-weight: bold; }
|
|
687
|
+
```
|
|
688
|
+
|
|
689
|
+
## 🕸️ Crawling the graph — `crawl.js`
|
|
690
|
+
|
|
691
|
+
MD-LD documents link to each other with ordinary Markdown links, so the graph
|
|
692
|
+
is already on the wire — you just need to follow it. `crawl.js` is a
|
|
693
|
+
**zero-dependency crawler primitive** that does exactly that: fetch raw text,
|
|
694
|
+
extract links, recurse. It knows nothing about MD-LD — the parser is applied at
|
|
695
|
+
the call site, which keeps the core pure and the crawler reusable for *any*
|
|
696
|
+
linked text format.
|
|
697
|
+
|
|
698
|
+
New in **v1.0.9**. Ships as a separate entry point (`mdld-parse/crawl`) and is
|
|
699
|
+
self-contained enough to load straight from the browser console — no bundler,
|
|
700
|
+
no build step:
|
|
701
|
+
|
|
702
|
+
```js
|
|
703
|
+
// Browser console, on any MD-LD site
|
|
704
|
+
const { crawl } = await import('/crawl.js');
|
|
705
|
+
const { pages, errors } = await crawl('/index.md', { sameOrigin: true });
|
|
706
|
+
console.table(pages.map(p => ({ url: p.url, depth: p.depth, links: p.links.length })));
|
|
707
|
+
```
|
|
708
|
+
|
|
709
|
+
```js
|
|
710
|
+
// Node ≥ 18 / Deno / Bun
|
|
711
|
+
import { crawl } from 'mdld-parse/crawl';
|
|
712
|
+
const { pages } = await crawl('https://mdld.js.org/index.md', { maxPages: 25 });
|
|
713
|
+
```
|
|
714
|
+
|
|
715
|
+
Full pipeline — crawl the graph, parse every document into its own named graph:
|
|
716
|
+
|
|
717
|
+
```js
|
|
718
|
+
import { crawl } from 'mdld-parse/crawl';
|
|
719
|
+
import { parse, deconstruct } from 'mdld-parse';
|
|
720
|
+
|
|
721
|
+
const { pages } = await crawl('/index.md', { sameOrigin: true, cache: myCache });
|
|
722
|
+
|
|
723
|
+
const results = pages.map(p => parse({
|
|
724
|
+
text: p.kind === 'html' ? deconstruct(p.text) : p.text, // HTML is a first-class transport
|
|
725
|
+
graph: p.finalUrl, // redirect-correct document identity
|
|
726
|
+
}));
|
|
727
|
+
const quads = results.flatMap(r => r.quads); // the whole crawled graph, provenance intact
|
|
728
|
+
```
|
|
729
|
+
|
|
730
|
+
### What it extracts
|
|
731
|
+
|
|
732
|
+
Both scanners run on **every** document — inline HTML is valid CommonMark, so
|
|
733
|
+
documents are routinely mixed:
|
|
734
|
+
|
|
735
|
+
- **Markdown**: inline links & images `[text](url)`, reference definitions
|
|
736
|
+
`[label]: url`, autolinks `<https://…>`
|
|
737
|
+
- **HTML**: `<a href>` — comment-aware, quote-aware, entities decoded;
|
|
738
|
+
`<script>`/`<style>` contents are skipped
|
|
739
|
+
|
|
740
|
+
Extraction is character-scanned (no regex in hot paths) and context-aware:
|
|
741
|
+
fenced code blocks, code spans and HTML comments never yield phantom links.
|
|
742
|
+
|
|
743
|
+
### Options
|
|
744
|
+
|
|
745
|
+
| Option | Default | Description |
|
|
746
|
+
| --- | --- | --- |
|
|
747
|
+
| `maxDepth` | `5` | Maximum link distance from the start document |
|
|
748
|
+
| `maxPages` | `100` | Hard cap on fetched pages — untrusted graphs can't balloon the crawl |
|
|
749
|
+
| `concurrency` | `5` | Maximum in-flight fetches |
|
|
750
|
+
| `sameOrigin` | `false` | Restrict the crawl to the start document's origin |
|
|
751
|
+
| `cache` | `memoryCache()` | Any `{ get, set, delete? }` adapter |
|
|
752
|
+
| `fetchFn` | `globalThis.fetch` | Wrap for retries, auth, proxies, rate limits |
|
|
753
|
+
| `accept` | docs extensions | `(url) => boolean` target filter |
|
|
754
|
+
| `onPage` | — | Per-page callback — parse incrementally while the crawl is in flight |
|
|
755
|
+
| `signal` | — | `AbortSignal` — cancels and resolves with partial results |
|
|
756
|
+
|
|
757
|
+
### Cache
|
|
758
|
+
|
|
759
|
+
Caching is a three-method interface — bring your own storage:
|
|
760
|
+
|
|
761
|
+
```js
|
|
762
|
+
const cache = {
|
|
763
|
+
async get(url) { /* → { text, etag, lastModified, ... } | null */ },
|
|
764
|
+
async set(url, entry) { /* persist */ },
|
|
765
|
+
async delete(url) { /* evict */ },
|
|
766
|
+
};
|
|
767
|
+
```
|
|
768
|
+
|
|
769
|
+
Conditional requests are automatic: stored `ETag` / `Last-Modified` are re-sent
|
|
770
|
+
as validators, `304` responses cost zero bytes and zero re-parse, and on 5xx or
|
|
771
|
+
network failure the cached copy is served (stale-if-error). Cache failures are
|
|
772
|
+
swallowed — a broken adapter (e.g. Safari private mode) degrades to "no cache",
|
|
773
|
+
never to a failed crawl. Drop-in adapters for IndexedDB, localStorage and Node
|
|
774
|
+
JSON-file caching live in `docs/cache-adapters.md`.
|
|
775
|
+
|
|
776
|
+
### Guarantees
|
|
777
|
+
|
|
778
|
+
- **Deterministic** — pages return in BFS discovery order, never completion order
|
|
779
|
+
- **Bounded** — `maxDepth`, `maxPages`, http(s)-only protocol allow-list, binary content-types rejected
|
|
780
|
+
- **Partial-result-safe** — errors are collected in `result.errors`; the crawl continues
|
|
781
|
+
- **Cancellable** — `AbortSignal` yields `{ aborted: true }` plus everything fetched so far
|
|
532
782
|
|
|
533
783
|
### Utility Functions
|
|
534
784
|
|
|
@@ -641,88 +891,6 @@ Comprehensive test suite covering:
|
|
|
641
891
|
- Origin tracking and provenance
|
|
642
892
|
|
|
643
893
|
|
|
644
|
-
## 📚 Documentation Hub
|
|
645
|
-
|
|
646
|
-
- **📋 [Specification](./spec/index.md)** — Formal specification and test suite
|
|
647
|
-
- [Formal Specification](./spec/Spec.md)
|
|
648
|
-
- [Compact Spec](./spec/Spec-compact.md)
|
|
649
|
-
- [Ultra Compact Spec](./spec/Spec-ultra.md)
|
|
650
|
-
|
|
651
|
-
- **📚 [Grammar](./grammar/index.md)** — EBNF+TextMate grammar specifications
|
|
652
|
-
- [The TextMate grammar](./grammar/mdld.tmLanguage.json)
|
|
653
|
-
- [ISO 14977 EBNF grammar](./grammar/mdld.ebnf)
|
|
654
|
-
|
|
655
|
-
- **📖 [Documentation](./docs/index.md)** — Complete documentation with guides and references
|
|
656
|
-
- [MD-LD Authoring Guide](./docs/Guide.md)
|
|
657
|
-
- [One Page Guide](./docs/one-page.md)
|
|
658
|
-
- [Semantic Infrastructure as Readable Text](./docs/Semantic-Text.md)
|
|
659
|
-
- [Elevated Statements](./docs/statements.md)
|
|
660
|
-
- [Primary Metadata System](./docs/primary-subject.md)
|
|
661
|
-
- [Origin System](./docs/origin.md)
|
|
662
|
-
- [Polarity & Retraction](./docs/polarity.md)
|
|
663
|
-
- [Subject System](./docs/Subject.md)
|
|
664
|
-
- [API Reference](./docs/API.md)
|
|
665
|
-
- [Generate: Quads to MDLD](./docs/generate.md)
|
|
666
|
-
- [Render MDLD to HTML and deconstruct it back](./docs/render.md)
|
|
667
|
-
- [Diff Documents](./docs/diff.md)
|
|
668
|
-
- [Syntax Reference](./docs/Syntax.md)
|
|
669
|
-
- [Architecture & Design](./docs/Architecture.md)
|
|
670
|
-
- [Parser Architecture](./docs/Parser.md)
|
|
671
|
-
- [Performance Benchmarks](./docs/Performance.md)
|
|
672
|
-
- [Token Efficiency](./docs/Token-Efficiency.md)
|
|
673
|
-
- [Knowledge Round Trip](./docs/Knowledge-Round-Trip.md)
|
|
674
|
-
- [Human-Scale Software & Semantic Infrastructure](./docs/Human-Scale.md)
|
|
675
|
-
- [Quad[] as a Universal Semantic Runtime](./docs/quad-runtime.md)
|
|
676
|
-
- [Use Cases](./docs/Use-Cases.md)
|
|
677
|
-
|
|
678
|
-
- **🎯 [Examples](./examples/index.md)** — Real-world MD-LD examples and use cases
|
|
679
|
-
- [Minimal](./examples/demo/minimal.md)
|
|
680
|
-
- [One-Page Guide](./examples/one-page.md)
|
|
681
|
-
- [Few-Shot Examples](./examples/few-shot.md)
|
|
682
|
-
- [Journal](./examples/demo/journal.md)
|
|
683
|
-
- [Tasks](./examples/demo/tasks.md)
|
|
684
|
-
- [Medical AI Research](./examples/demo/research.md)
|
|
685
|
-
- [Cookbook](./examples/Cookbook.md)
|
|
686
|
-
- [Website Redesign Project](./examples/demo/project.md)
|
|
687
|
-
- [Task Management System](./examples/demo/tasks.md)
|
|
688
|
-
- [Cassini-Huygens Mission](./examples/demo/space-mission.md)
|
|
689
|
-
- [RDF Fundamentals](./examples/demo/rdf.md)
|
|
690
|
-
- [Statements Golden Graph](./examples/statements.md)
|
|
691
|
-
- [PROV-O Patterns](./examples/demo/prov.md)
|
|
692
|
-
- [SHACL Validation](./examples/demo/shacl.md)
|
|
693
|
-
- [XSD Datatypes](./examples/demo/xsd.md)
|
|
694
|
-
- [Status & SHACL Validation](./examples/Status-SHACL.md)
|
|
695
|
-
- [LLM Time Workflow](./examples/llm-time-workflow.md)
|
|
696
|
-
- [Research Workflow](./examples/workflow.md)
|
|
697
|
-
- [Dogfood](./examples/dogfood.md)
|
|
698
|
-
|
|
699
|
-
- **💻 [Reference server](./server/index.md) - a Node.js implementation of the server for git-powered semantic workflows and publishing
|
|
700
|
-
- [Source code](./server/mdld-server.js)
|
|
701
|
-
|
|
702
|
-
- **🧩 [Ontologies](./ontologies/index.md)** — W3C and related standard ontologies used in RDF
|
|
703
|
-
- [RDF — Resource Description Framework](./ontologies/rdf.md)
|
|
704
|
-
- [RDFS — RDF Schema](./ontologies/rdfs.md)
|
|
705
|
-
- [SHACL — Shapes Constraint Language](./ontologies/shacl.md)
|
|
706
|
-
- [PROV-O — W3C Provenance Ontology](./ontologies/prov-o.md)
|
|
707
|
-
- [XSD — XML Schema Definition Datatypes](./ontologies/xsd.md)
|
|
708
|
-
- [DCTERMS — Dublin Core Metadata Terms](./ontologies/dcterms.md)
|
|
709
|
-
- [DCAT — Data Catalog Vocabulary](./ontologies/dcat.md)
|
|
710
|
-
- [FOAF — Friend of a Friend](./ontologies/foaf.md)
|
|
711
|
-
- [VCard](./ontologies/vcard.md)
|
|
712
|
-
- [Time — OWL-Time Ontology](./ontologies/time.md)
|
|
713
|
-
- [Activity Streams 2.0](./ontologies/activity-streams-2.md)
|
|
714
|
-
- [SKOS — Simple Knowledge Organization System](./ontologies/skos.md)
|
|
715
|
-
- [Schema.org](./ontologies/schemaorg.md)
|
|
716
|
-
- [CIDOC CRM — Conceptual Reference Model](./ontologies/cidoc_crm.md)
|
|
717
|
-
- [SOSA — Sensor, Observation, Sample, and Actuator](./ontologies/sosa.md)
|
|
718
|
-
- [QUDT — Quantities, Units, Dimensions and Types](./ontologies/qudt.md)
|
|
719
|
-
- [GOLD — General Ontology for Linguistic Description](./ontologies/gold.md)
|
|
720
|
-
- [LexInfo — Lexical Information Ontology](./ontologies/lexinfo.md)
|
|
721
|
-
- [OWL — Web Ontology Language](./ontologies/owl.md)
|
|
722
|
-
- [P-PLAN — Plan Ontology](./ontologies/p-plan.md)
|
|
723
|
-
- [Hydra — Hypermedia API Vocabulary](./ontologies/hydra.md)
|
|
724
|
-
- [Web Annotations](./ontologies/web-annotations.md)
|
|
725
|
-
|
|
726
894
|
## Governance
|
|
727
895
|
|
|
728
896
|
MD-LD is a craft project. Its coherence comes from a single evolving understanding of how semantic text should work — not from consensus, but from sustained attention to the same problem over time.
|