@kenjura/ursa 0.96.0 → 0.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +72 -16
  3. package/bin/ursa.js +14 -1
  4. package/meta/templates/default-template/menu.js +18 -1
  5. package/meta/templates/default-template/search.js +11 -0
  6. package/meta/templates/default-template/widgets.js +4 -0
  7. package/package.json +1 -2
  8. package/src/dev.js +13 -23
  9. package/src/helper/__test__/contentHash.test.js +16 -6
  10. package/src/helper/assetBundler.js +93 -19
  11. package/src/helper/automenu.js +36 -11
  12. package/src/helper/build/__test__/autoIndex.test.js +2 -132
  13. package/src/helper/build/__test__/graph.test.js +259 -3
  14. package/src/helper/build/__test__/pass.test.js +553 -0
  15. package/src/helper/build/autoIndex.js +2 -371
  16. package/src/helper/build/excludeFilter.js +1 -2
  17. package/src/helper/build/footer.js +27 -14
  18. package/src/helper/build/graph.js +575 -152
  19. package/src/helper/build/index.js +0 -2
  20. package/src/helper/build/metadata.js +19 -5
  21. package/src/helper/build/pass.js +497 -0
  22. package/src/helper/build/precedence.js +174 -0
  23. package/src/helper/build/site.js +1270 -0
  24. package/src/helper/build/templates.js +1 -2
  25. package/src/helper/build/tracedFs.js +247 -0
  26. package/src/helper/contentHash.js +0 -78
  27. package/src/helper/customMenu.js +1 -1
  28. package/src/helper/fileRenderer.js +119 -111
  29. package/src/helper/findScriptJs.js +1 -1
  30. package/src/helper/findStyleCss.js +1 -1
  31. package/src/helper/folderConfig.js +7 -18
  32. package/src/helper/fullTextIndex.js +41 -29
  33. package/src/helper/imageProcessor.js +45 -0
  34. package/src/helper/linkValidator.js +118 -127
  35. package/src/helper/mdxRenderer.js +27 -5
  36. package/src/helper/menuLabels.js +30 -5
  37. package/src/helper/whitelistFilter.js +1 -2
  38. package/src/jobs/generate.js +67 -1829
  39. package/src/serve.js +317 -697
  40. package/src/helper/__test__/dependencyTracker.test.js +0 -157
  41. package/src/helper/build/cacheBust.js +0 -141
  42. package/src/helper/build/navCache.js +0 -145
  43. package/src/helper/build/watchCache.js +0 -33
  44. package/src/helper/dependencyTracker.js +0 -384
@@ -1,9 +1,7 @@
1
1
  import { join } from "path";
2
- import { mkdtemp, mkdir, writeFile, rm, readFile } from "fs/promises";
3
- import { existsSync } from "fs";
2
+ import { mkdtemp, mkdir, writeFile, rm } from "fs/promises";
4
3
  import { tmpdir } from "os";
5
- import { generateAutoIndices, generateAutoIndexHtmlFromSource } from "../autoIndex.js";
6
- import { clearConfigCache } from "../../folderConfig.js";
4
+ import { generateAutoIndexHtmlFromSource } from "../autoIndex.js";
7
5
 
8
6
  let tempDir;
9
7
  let source;
@@ -19,82 +17,6 @@ afterEach(async () => {
19
17
  await rm(tempDir, { recursive: true, force: true });
20
18
  });
21
19
 
22
- const TEMPLATE =
23
- "<html><head>${styleLink}</head><body>${menu}${body}${footer}${customScript}</body></html>";
24
-
25
- function makeProgress() {
26
- const logs = [];
27
- return {
28
- logs,
29
- log: (msg) => logs.push(msg),
30
- status: () => {},
31
- done: () => {},
32
- };
33
- }
34
-
35
- function runAutoIndices(directories, generatedArticles, progress) {
36
- return generateAutoIndices(
37
- output,
38
- directories,
39
- source,
40
- { "default-template": TEMPLATE },
41
- "",
42
- "",
43
- generatedArticles,
44
- new Set(),
45
- new Set(),
46
- "20260101000000",
47
- progress,
48
- null
49
- );
50
- }
51
-
52
- describe("generateAutoIndices with empty source folders", () => {
53
- it("skips output directories that were never created instead of logging an error", async () => {
54
- // Source has an empty folder (guides) and a folder with a document (docs).
55
- // Only docs produced output files, so output/guides does not exist.
56
- await mkdir(join(source, "guides"));
57
- await mkdir(join(source, "docs"));
58
- await writeFile(join(source, "docs", "hello.md"), "# Hello\n\nWorld\n");
59
- await mkdir(join(output, "docs"));
60
- await writeFile(join(output, "docs", "hello.html"), "<html><body>Hello</body></html>");
61
-
62
- const progress = makeProgress();
63
- await runAutoIndices(
64
- [join(source, "guides"), join(source, "docs")],
65
- [join(source, "docs", "hello.md")],
66
- progress
67
- );
68
-
69
- const errors = progress.logs.filter((m) => /Error generating auto-index/i.test(m));
70
- expect(errors).toEqual([]);
71
- // The missing output directory is skipped, not created
72
- expect(existsSync(join(output, "guides"))).toBe(false);
73
- expect(existsSync(join(output, "guides", "index.html"))).toBe(false);
74
- });
75
-
76
- it("still generates auto-indices for folders that produced output", async () => {
77
- await mkdir(join(source, "guides"));
78
- await mkdir(join(source, "docs"));
79
- await writeFile(join(source, "docs", "hello.md"), "# Hello\n\nWorld\n");
80
- await mkdir(join(output, "docs"));
81
- await writeFile(join(output, "docs", "hello.html"), "<html><body>Hello</body></html>");
82
-
83
- const progress = makeProgress();
84
- await runAutoIndices(
85
- [join(source, "guides"), join(source, "docs")],
86
- [join(source, "docs", "hello.md")],
87
- progress
88
- );
89
-
90
- // Root and docs both exist in output, so both get an index.html
91
- const docsIndex = await readFile(join(output, "docs", "index.html"), "utf8");
92
- expect(docsIndex).toContain('<a href="hello.html">');
93
- const rootIndex = await readFile(join(output, "index.html"), "utf8");
94
- expect(rootIndex).toContain('<a href="docs/index.html">');
95
- });
96
- });
97
-
98
20
  describe("auto-index naming matches the automenu", () => {
99
21
  it("uses menu-label from a folder's index frontmatter, and config.json when there is no index", async () => {
100
22
  // bnw has a real index.md carrying the label; hfr has no index at all, so
@@ -140,35 +62,9 @@ describe("auto-index naming matches the automenu", () => {
140
62
 
141
63
  expect(html).toContain('<a href="SoL/index.html">SoL</a>');
142
64
  });
143
-
144
- it("labels generated index pages with the folder's menu-label", async () => {
145
- // No index.md content, so this folder gets an auto-generated index.html;
146
- // its <h1> should use the label rather than the raw folder name.
147
- await mkdir(join(source, "bnw"));
148
- await writeFile(
149
- join(source, "bnw", "config.json"),
150
- JSON.stringify({ label: "BNW - Brave New World" })
151
- );
152
- await writeFile(join(source, "bnw", "quests.md"), "# Quests\n");
153
- await mkdir(join(output, "bnw"));
154
- await writeFile(join(output, "bnw", "quests.html"), "<html><body>Quests</body></html>");
155
-
156
- await runAutoIndices([join(source, "bnw")], [join(source, "bnw", "quests.md")], makeProgress());
157
-
158
- const bnwIndex = await readFile(join(output, "bnw", "index.html"), "utf8");
159
- expect(bnwIndex).toContain("<h1>BNW - Brave New World</h1>");
160
- const rootIndex = await readFile(join(output, "index.html"), "utf8");
161
- expect(rootIndex).toContain('<a href="bnw/index.html">BNW - Brave New World</a>');
162
- });
163
65
  });
164
66
 
165
67
  describe("folders ignored via config.json { hidden: true }", () => {
166
- beforeEach(() => {
167
- // getFolderConfig memoizes per absolute path; temp dirs are unique per
168
- // test, but clearing keeps the cache from growing across the suite.
169
- clearConfigCache();
170
- });
171
-
172
68
  it("omits a hidden folder from an auto-index built from source", async () => {
173
69
  await mkdir(join(source, "_art"), { recursive: true });
174
70
  await writeFile(join(source, "_art", "config.json"), JSON.stringify({ hidden: true }));
@@ -183,32 +79,6 @@ describe("folders ignored via config.json { hidden: true }", () => {
183
79
  expect(html).not.toContain("prompts");
184
80
  });
185
81
 
186
- it("omits a hidden folder even when stale output for it still exists", async () => {
187
- // A folder generated before it was hidden leaves files behind in output.
188
- // The listing is built from output, so without a source-side check the
189
- // hidden folder would reappear in the index.
190
- await mkdir(join(source, "_art"), { recursive: true });
191
- await writeFile(join(source, "_art", "config.json"), JSON.stringify({ hidden: true }));
192
- await mkdir(join(source, "people"), { recursive: true });
193
- await writeFile(join(source, "people", "alice.md"), "# Alice\n");
194
-
195
- await mkdir(join(output, "_art"), { recursive: true });
196
- await writeFile(join(output, "_art", "prompts.html"), "<html><body>stale</body></html>");
197
- await mkdir(join(output, "people"), { recursive: true });
198
- await writeFile(join(output, "people", "alice.html"), "<html><body>Alice</body></html>");
199
-
200
- const progress = makeProgress();
201
- await runAutoIndices(
202
- [source, join(source, "people")],
203
- [join(source, "people", "alice.md")],
204
- progress
205
- );
206
-
207
- const index = await readFile(join(output, "index.html"), "utf8");
208
- expect(index).toContain("people");
209
- expect(index).not.toContain("_art");
210
- });
211
-
212
82
  it("does not count documents inside a hidden subfolder when deciding a folder has content", async () => {
213
83
  // `notes` holds nothing but a hidden subfolder, so it produces no pages
214
84
  // and must not be linked as though it did.
@@ -3,13 +3,17 @@ import { mkdtemp, writeFile, rm, unlink, readFile } from "fs/promises";
3
3
  import { tmpdir } from "os";
4
4
  import {
5
5
  BuildGraph,
6
+ GRAPH_SCHEMA_VERSION,
6
7
  GraphComputeError,
8
+ dirNodeId,
7
9
  fileNodeId,
8
10
  lookupNodeId,
9
11
  loadGraph,
10
12
  saveGraph,
11
13
  getGraphPath,
12
14
  } from "../graph.js";
15
+ import * as tfs from "../tracedFs.js";
16
+ import { mkdir } from "fs/promises";
13
17
 
14
18
  let tempDir;
15
19
  beforeEach(async () => {
@@ -353,7 +357,7 @@ describe("persistence", () => {
353
357
  await g1.build(["c"]);
354
358
  expect(await saveGraph(tempDir, g1)).toBe(true);
355
359
  const onDisk = JSON.parse(await readFile(getGraphPath(tempDir), "utf8"));
356
- expect(onDisk.version).toBe(1);
360
+ expect(onDisk.version).toBe(GRAPH_SCHEMA_VERSION);
357
361
 
358
362
  const g2 = new BuildGraph();
359
363
  const counters2 = { b: 0, c: 0 };
@@ -458,7 +462,7 @@ describe("engine behavior", () => {
458
462
  });
459
463
  await graph.build(["parent"]);
460
464
 
461
- graph.removeNode("dep");
465
+ await graph.removeNode("dep");
462
466
  const r = await graph.build(["parent"]);
463
467
  expect(r.results.get("parent")).toBe("alone");
464
468
  });
@@ -471,7 +475,7 @@ describe("engine behavior", () => {
471
475
  graph.node("pageB", (ctx) => ctx.read(p("b.txt")));
472
476
  await graph.build(["pageA", "pageB"]);
473
477
 
474
- graph.gc(new Set(["pageA"]));
478
+ await graph.gc(new Set(["pageA"]));
475
479
  expect(graph.edges.has("pageB")).toBe(false);
476
480
  expect(graph.fingerprints.has("pageB")).toBe(false);
477
481
  expect(graph.fingerprints.has(fileNodeId(p("b.txt")))).toBe(false);
@@ -527,3 +531,255 @@ describe("engine behavior", () => {
527
531
  expect(stats.failed).toBe(0);
528
532
  });
529
533
  });
534
+
535
+ // ---------------------------------------------------------------------------
536
+ // Directory leaves
537
+ // ---------------------------------------------------------------------------
538
+ describe("directory leaves", () => {
539
+ it("adding, removing or renaming an entry changes the listing; editing content does not", async () => {
540
+ await mkdir(p("d"));
541
+ await writeFile(p("d/a.md"), "one");
542
+ const graph = new BuildGraph();
543
+ let count = 0;
544
+ graph.node("names", async (ctx) => {
545
+ count++;
546
+ return (await ctx.listDir(p("d"))).map((e) => e.name);
547
+ });
548
+ const r1 = await graph.build(["names"]);
549
+ expect(r1.results.get("names")).toEqual(["a.md"]);
550
+
551
+ // Content edit: the listing is unchanged
552
+ await writeFile(p("d/a.md"), "two");
553
+ graph.invalidatePath(p("d/a.md"));
554
+ await graph.build(["names"]);
555
+ expect(count).toBe(1);
556
+
557
+ // New entry
558
+ await writeFile(p("d/b.md"), "x");
559
+ graph.invalidatePath(p("d/b.md"));
560
+ const r2 = await graph.build(["names"]);
561
+ expect(r2.results.get("names")).toEqual(["a.md", "b.md"]);
562
+ expect(count).toBe(2);
563
+
564
+ // Scratch files are invisible
565
+ await writeFile(p("d/.b.md.swp"), "x");
566
+ await writeFile(p("d/4913"), "x");
567
+ graph.invalidatePath(p("d/4913"));
568
+ await graph.build(["names"]);
569
+ expect(count).toBe(2);
570
+ });
571
+
572
+ it("a missing directory lists as [] and its creation is observed", async () => {
573
+ const graph = new BuildGraph();
574
+ graph.node("names", async (ctx) => (await ctx.listDir(p("later"))).map((e) => e.name));
575
+ const r1 = await graph.build(["names"]);
576
+ expect(r1.results.get("names")).toEqual([]);
577
+ await mkdir(p("later"));
578
+ await writeFile(p("later/x.md"), "x");
579
+ graph.invalidateSubtree(p("later"));
580
+ const r2 = await graph.build(["names"]);
581
+ expect(r2.results.get("names")).toEqual(["x.md"]);
582
+ });
583
+ });
584
+
585
+ // ---------------------------------------------------------------------------
586
+ // Traced fs: helpers that read the disk directly still record edges
587
+ // ---------------------------------------------------------------------------
588
+ describe("traced filesystem", () => {
589
+ it("records sync reads made by nested helpers as edges of the computing node", async () => {
590
+ await writeFile(p("cfg.json"), '{"label":"A"}');
591
+ const helper = () => (tfs.existsSync(p("cfg.json")) ? JSON.parse(tfs.readFileSync(p("cfg.json"), "utf8")).label : "none");
592
+ const graph = new BuildGraph();
593
+ let count = 0;
594
+ graph.node("label", async () => {
595
+ count++;
596
+ return helper();
597
+ });
598
+ const r1 = await graph.build(["label"]);
599
+ expect(r1.results.get("label")).toBe("A");
600
+ expect(graph.edges.get("label").has(fileNodeId(p("cfg.json")))).toBe(true);
601
+ expect(graph.edges.get("label").has(lookupNodeId(p("cfg.json")))).toBe(true);
602
+
603
+ await writeFile(p("cfg.json"), '{"label":"B"}');
604
+ graph.invalidatePath(p("cfg.json"));
605
+ const r2 = await graph.build(["label"]);
606
+ expect(r2.results.get("label")).toBe("B");
607
+ expect(count).toBe(2);
608
+
609
+ await unlink(p("cfg.json"));
610
+ graph.invalidatePath(p("cfg.json"));
611
+ const r3 = await graph.build(["label"]);
612
+ expect(r3.results.get("label")).toBe("none");
613
+ });
614
+
615
+ it("attributes concurrent computes to the right node", async () => {
616
+ await writeFile(p("x.txt"), "x");
617
+ await writeFile(p("y.txt"), "y");
618
+ const graph = new BuildGraph();
619
+ graph.node("x", async () => {
620
+ await new Promise((r) => setTimeout(r, 5));
621
+ return tfs.readFileSync(p("x.txt"), "utf8");
622
+ });
623
+ graph.node("y", async () => {
624
+ await new Promise((r) => setTimeout(r, 1));
625
+ return tfs.readFileSync(p("y.txt"), "utf8");
626
+ });
627
+ await graph.build(["x", "y"], { concurrency: 2 });
628
+ expect([...graph.edges.get("x").keys()]).toEqual([fileNodeId(p("x.txt"))]);
629
+ expect([...graph.edges.get("y").keys()]).toEqual([fileNodeId(p("y.txt"))]);
630
+ });
631
+ });
632
+
633
+ // ---------------------------------------------------------------------------
634
+ // Concurrency
635
+ // ---------------------------------------------------------------------------
636
+ describe("concurrent roots", () => {
637
+ it("computes a shared dependency once when two roots demand it at the same time", async () => {
638
+ await writeFile(p("a.txt"), "a");
639
+ const graph = new BuildGraph();
640
+ let shared = 0;
641
+ graph.node("shared", async (ctx) => {
642
+ shared++;
643
+ await new Promise((r) => setTimeout(r, 5));
644
+ return ctx.read(p("a.txt"));
645
+ });
646
+ graph.node("r1", async (ctx) => "1:" + (await ctx.get("shared")));
647
+ graph.node("r2", async (ctx) => "2:" + (await ctx.get("shared")));
648
+ const r = await graph.build(["r1", "r2"], { concurrency: 2 });
649
+ expect(r.ok).toBe(true);
650
+ expect(shared).toBe(1);
651
+ });
652
+
653
+ it("still detects a real cycle under concurrency", async () => {
654
+ const graph = new BuildGraph();
655
+ graph.node("a", (ctx) => ctx.get("b"));
656
+ graph.node("b", (ctx) => ctx.get("a"));
657
+ const r = await graph.build(["a"], { concurrency: 2 });
658
+ expect(r.ok).toBe(false);
659
+ expect(String(r.errors.get("a"))).toMatch(/cycle/i);
660
+ });
661
+ });
662
+
663
+ // ---------------------------------------------------------------------------
664
+ // Ownership and orphan deletion
665
+ // ---------------------------------------------------------------------------
666
+ describe("output ownership", () => {
667
+ it("hands files a node stops owning to onOrphans, and all of them on removal", async () => {
668
+ await writeFile(p("a.txt"), "a");
669
+ const orphaned = [];
670
+ const graph = new BuildGraph({ onOrphans: (paths) => { orphaned.push(...paths); } });
671
+ let extra = true;
672
+ graph.node("out", async (ctx) => {
673
+ const v = await ctx.read(p("a.txt"));
674
+ ctx.own("/out/a.html");
675
+ if (extra) ctx.own("/out/a.xml");
676
+ return v;
677
+ });
678
+ await graph.build(["out"]);
679
+ expect(graph.ownerOfPath("/out/a.xml")).toBe("out");
680
+
681
+ extra = false;
682
+ await writeFile(p("a.txt"), "b");
683
+ graph.invalidatePath(p("a.txt"));
684
+ await graph.build(["out"]);
685
+ expect(orphaned).toEqual(["/out/a.xml"]);
686
+ expect(graph.ownerOfPath("/out/a.xml")).toBe(null);
687
+
688
+ await graph.removeNode("out");
689
+ expect(orphaned).toEqual(["/out/a.xml", "/out/a.html"]);
690
+ });
691
+
692
+ it("a path whose ownership moves to another node is not an orphan", async () => {
693
+ const orphaned = [];
694
+ const graph = new BuildGraph({ onOrphans: (paths) => { orphaned.push(...paths); } });
695
+ graph.node("first", async (ctx) => { ctx.own("/index.html"); return 1; });
696
+ graph.node("second", async (ctx) => { ctx.own("/index.html"); return 2; });
697
+ await graph.build(["first"]);
698
+ await graph.build(["second"]);
699
+ await graph.removeNode("first");
700
+ expect(orphaned).toEqual([]);
701
+ expect(graph.ownerOfPath("/index.html")).toBe("second");
702
+ });
703
+
704
+ it("persists ownership", async () => {
705
+ const g1 = new BuildGraph();
706
+ g1.node("n", async (ctx) => { ctx.own("/n.html"); return 1; });
707
+ await g1.build(["n"]);
708
+ const g2 = new BuildGraph();
709
+ expect(g2.load(g1.serialize())).toBe(true);
710
+ expect(g2.ownedBy("n")).toEqual(["/n.html"]);
711
+ expect(g2.ownerOfPath("/n.html")).toBe("n");
712
+ });
713
+ });
714
+
715
+ // ---------------------------------------------------------------------------
716
+ // Node families, relocatable ids, dirty sets, explain
717
+ // ---------------------------------------------------------------------------
718
+ describe("families and roots", () => {
719
+ it("resolves node families on demand", async () => {
720
+ await writeFile(p("a.txt"), "A");
721
+ const graph = new BuildGraph();
722
+ graph.resolver((id) => {
723
+ if (id.startsWith("upper:")) {
724
+ const file = id.slice("upper:".length);
725
+ return { fn: async (ctx) => (await ctx.read(p(file))).toUpperCase() };
726
+ }
727
+ return null;
728
+ });
729
+ graph.node("page", async (ctx) => "<" + (await ctx.get("upper:a.txt")) + ">");
730
+ const r = await graph.build(["page"]);
731
+ expect(r.results.get("page")).toBe("<A>");
732
+ expect(graph.hasNode("upper:a.txt")).toBe(true);
733
+ expect(graph.hasNode("nope:x")).toBe(false);
734
+ });
735
+
736
+ it("stores leaf ids relative to a root and resolves them back", async () => {
737
+ await mkdir(p("src"));
738
+ await writeFile(p("src/a.txt"), "A");
739
+ const g1 = new BuildGraph();
740
+ g1.setRoots({ S: p("src") });
741
+ g1.node("n", (ctx) => ctx.read(p("src/a.txt")));
742
+ await g1.build(["n"]);
743
+ expect([...g1.edges.get("n").keys()]).toEqual([fileNodeId("$S/a.txt")]);
744
+
745
+ // Relocate: the same tree under another root still verifies clean
746
+ await mkdir(p("moved"));
747
+ await writeFile(p("moved/a.txt"), "A");
748
+ const g2 = new BuildGraph();
749
+ g2.setRoots({ S: p("moved") });
750
+ g2.node("n", (ctx) => ctx.read(p("moved/a.txt")));
751
+ expect(g2.load(g1.serialize())).toBe(true);
752
+ await g2.scanLeaves();
753
+ const r = await g2.build(["n"]);
754
+ expect(r.computed.size).toBe(0);
755
+ });
756
+
757
+ it("dependents() gives the transitive dirty set of changed leaves", async () => {
758
+ await writeFile(p("a.txt"), "a");
759
+ await writeFile(p("b.txt"), "b");
760
+ const graph = new BuildGraph();
761
+ graph.node("A", (ctx) => ctx.read(p("a.txt")));
762
+ graph.node("B", (ctx) => ctx.read(p("b.txt")));
763
+ graph.node("AB", async (ctx) => (await ctx.get("A")) + (await ctx.get("B")));
764
+ graph.node("A2", async (ctx) => (await ctx.get("A")) + "!");
765
+ await graph.build(["AB", "A2"]);
766
+ await writeFile(p("b.txt"), "bb");
767
+ graph.invalidatePath(p("b.txt"));
768
+ const changed = await graph.refreshStale();
769
+ expect(changed).toEqual([fileNodeId(p("b.txt"))]);
770
+ expect([...graph.dependents(changed)].sort()).toEqual(["AB", "B"]);
771
+ });
772
+
773
+ it("records why each node recomputed", async () => {
774
+ await writeFile(p("a.txt"), "a");
775
+ const graph = new BuildGraph();
776
+ graph.node("A", (ctx) => ctx.read(p("a.txt")));
777
+ graph.node("AA", async (ctx) => (await ctx.get("A")) + "!");
778
+ await graph.build(["AA"]);
779
+ await writeFile(p("a.txt"), "b");
780
+ graph.invalidatePath(p("a.txt"));
781
+ await graph.build(["AA"]);
782
+ expect(graph.reasons.get("A")).toBe(fileNodeId(p("a.txt")));
783
+ expect(graph.reasons.get("AA")).toBe("A");
784
+ });
785
+ });