@dogsbay/site-check 0.0.0-stage → 0.2.0-beta.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -2
- package/dist/index.d.ts +24 -0
- package/dist/index.js +31 -0
- package/dist/lib/config-redirects.d.ts +22 -0
- package/dist/lib/config-redirects.js +62 -0
- package/dist/lib/page.d.ts +19 -0
- package/dist/lib/page.js +34 -0
- package/dist/lib/release.d.ts +118 -0
- package/dist/lib/release.js +206 -0
- package/dist/lib/validate-refs.d.ts +39 -0
- package/dist/lib/validate-refs.js +165 -0
- package/dist/links.d.ts +53 -0
- package/dist/links.js +109 -0
- package/dist/node/collect-assets.d.ts +7 -0
- package/dist/node/collect-assets.js +64 -0
- package/dist/node/crawler.d.ts +28 -0
- package/dist/node/crawler.js +80 -0
- package/dist/node/git-renames.d.ts +6 -0
- package/dist/node/git-renames.js +45 -0
- package/dist/node.d.ts +5 -0
- package/dist/node.js +4 -0
- package/dist/output/json.d.ts +7 -0
- package/dist/output/json.js +3 -0
- package/dist/output/release.d.ts +33 -0
- package/dist/output/release.js +79 -0
- package/dist/output/text.d.ts +6 -0
- package/dist/output/text.js +85 -0
- package/dist/parse.d.ts +2 -0
- package/dist/parse.js +9 -0
- package/dist/registry.d.ts +55 -0
- package/dist/registry.js +62 -0
- package/dist/rules/frontmatter/index.d.ts +4 -0
- package/dist/rules/frontmatter/index.js +15 -0
- package/dist/rules/frontmatter/valid.d.ts +2 -0
- package/dist/rules/frontmatter/valid.js +34 -0
- package/dist/rules/seo/canonical.d.ts +3 -0
- package/dist/rules/seo/canonical.js +124 -0
- package/dist/rules/seo/description-length.d.ts +16 -0
- package/dist/rules/seo/description-length.js +26 -0
- package/dist/rules/seo/description-missing.d.ts +15 -0
- package/dist/rules/seo/description-missing.js +24 -0
- package/dist/rules/seo/dist-meta.d.ts +19 -0
- package/dist/rules/seo/dist-meta.js +67 -0
- package/dist/rules/seo/duplicate-title.d.ts +15 -0
- package/dist/rules/seo/duplicate-title.js +47 -0
- package/dist/rules/seo/h1-brand-keyword.d.ts +2 -0
- package/dist/rules/seo/h1-brand-keyword.js +102 -0
- package/dist/rules/seo/index.d.ts +11 -0
- package/dist/rules/seo/index.js +68 -0
- package/dist/rules/seo/json-ld-parse.d.ts +13 -0
- package/dist/rules/seo/json-ld-parse.js +32 -0
- package/dist/rules/seo/json-ld-required-fields.d.ts +21 -0
- package/dist/rules/seo/json-ld-required-fields.js +116 -0
- package/dist/rules/seo/keyword-placement.d.ts +2 -0
- package/dist/rules/seo/keyword-placement.js +125 -0
- package/dist/rules/seo/sitemap.d.ts +55 -0
- package/dist/rules/seo/sitemap.js +366 -0
- package/dist/rules/seo/title-missing.d.ts +20 -0
- package/dist/rules/seo/title-missing.js +33 -0
- package/dist/rules/structure/anchors.d.ts +2 -0
- package/dist/rules/structure/anchors.js +133 -0
- package/dist/rules/structure/asset-refs.d.ts +23 -0
- package/dist/rules/structure/asset-refs.js +41 -0
- package/dist/rules/structure/heading-order.d.ts +2 -0
- package/dist/rules/structure/heading-order.js +51 -0
- package/dist/rules/structure/index.d.ts +11 -0
- package/dist/rules/structure/index.js +50 -0
- package/dist/rules/structure/internal-links.d.ts +22 -0
- package/dist/rules/structure/internal-links.js +28 -0
- package/dist/rules/structure/locale-coherence.d.ts +17 -0
- package/dist/rules/structure/locale-coherence.js +32 -0
- package/dist/rules/structure/namespace-coherence.d.ts +17 -0
- package/dist/rules/structure/namespace-coherence.js +39 -0
- package/dist/rules/structure/release-moves.d.ts +14 -0
- package/dist/rules/structure/release-moves.js +33 -0
- package/dist/rules/structure/unresolved-directives.d.ts +2 -0
- package/dist/rules/structure/unresolved-directives.js +108 -0
- package/dist/rules/structure/version-coherence.d.ts +25 -0
- package/dist/rules/structure/version-coherence.js +45 -0
- package/dist/run.d.ts +97 -0
- package/dist/run.js +103 -0
- package/dist/session.d.ts +55 -0
- package/dist/session.js +153 -0
- package/dist/types.d.ts +265 -0
- package/dist/types.js +1 -0
- package/package.json +46 -3
package/README.md
CHANGED
|
@@ -1,3 +1,22 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @dogsbay/site-check
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The site audit engine behind `dogsbay site check` (the Astro pipeline) and `dogsbay-ssg check`: rules over source pages, the whole corpus and built HTML, a runner, and text and JSON reports.
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
import { filterRules, formatText, registerSharedRules, reportExitCode, runAudit } from "@dogsbay/site-check";
|
|
7
|
+
import { crawlDist, distFileReader } from "@dogsbay/site-check/node";
|
|
8
|
+
|
|
9
|
+
registerSharedRules();
|
|
10
|
+
const report = runAudit({
|
|
11
|
+
rules: filterRules({}),
|
|
12
|
+
pages, nav, config,
|
|
13
|
+
distFiles: crawlDist("dist"),
|
|
14
|
+
readFile: distFileReader("dist"),
|
|
15
|
+
});
|
|
16
|
+
console.log(formatText(report));
|
|
17
|
+
process.exit(reportExitCode(report, false));
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The main entry is Worker-safe (no `node:` imports) when bundled with the `browser` or `workerd` condition, which selects cheerio's browser build. An in-memory build passes `parseHtmlFile(path, html)` for each page it emits and a `readFile` over its output instead of the `./node` helpers.
|
|
21
|
+
|
|
22
|
+
Design: [`docs-dev/site-check-architecture.md`](../../docs-dev/site-check-architecture.md).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export * from "./types.js";
|
|
2
|
+
export { registerRule, listRules, getRule, filterRules, _clearRegistry } from "./registry.js";
|
|
3
|
+
export type { FilterOptions } from "./registry.js";
|
|
4
|
+
export { runAudit, reportExitCode } from "./run.js";
|
|
5
|
+
export type { RunAuditInputs } from "./run.js";
|
|
6
|
+
export { formatJson } from "./output/json.js";
|
|
7
|
+
export { formatText } from "./output/text.js";
|
|
8
|
+
export type { FormatTextOptions } from "./output/text.js";
|
|
9
|
+
export { toReport } from "./run.js";
|
|
10
|
+
export { createAuditSession, slimPage } from "./session.js";
|
|
11
|
+
export type { AuditSession, AuditSessionOptions } from "./session.js";
|
|
12
|
+
export { parseHtmlFile } from "./parse.js";
|
|
13
|
+
export { validateRefs } from "./lib/validate-refs.js";
|
|
14
|
+
export { compareRelease, releasePage, releaseBaseline, normalizeRemoved, signature, similarity, treeText, MOVED_SIMILARITY, MIN_SHINGLES } from "./lib/release.js";
|
|
15
|
+
export type { ReleasePage, ReleaseEntry, ReleaseStatus, CompareReleaseInput, ReleaseComparison } from "./lib/release.js";
|
|
16
|
+
export { formatReleaseText, releaseRedirects, markWritten, releaseOpen, releaseExitCode, formatWriteSummary } from "./output/release.js";
|
|
17
|
+
export { addRedirectsToConfig, redirectsBlock } from "./lib/config-redirects.js";
|
|
18
|
+
export type { ConfigRedirect } from "./lib/config-redirects.js";
|
|
19
|
+
export type { RefIssue, ValidateRefsOptions, ValidateRefsResult } from "./lib/validate-refs.js";
|
|
20
|
+
export { registerSeoRules, _resetSeoRegistration } from "./rules/seo/index.js";
|
|
21
|
+
export { registerStructureRules, _resetStructureRegistration } from "./rules/structure/index.js";
|
|
22
|
+
export { registerFrontmatterRules, _resetFrontmatterRegistration } from "./rules/frontmatter/index.js";
|
|
23
|
+
/** Register every shared rule (SEO, structure and frontmatter). Idempotent. */
|
|
24
|
+
export declare function registerSharedRules(): void;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @dogsbay/site-check — the site audit engine shared by the Astro
|
|
3
|
+
* pipeline (`dogsbay site check`) and dogsbay-ssg (`dogsbay-ssg check`).
|
|
4
|
+
*
|
|
5
|
+
* This entry is Worker-safe: no `node:` imports. Reading a dist folder
|
|
6
|
+
* and collecting assets from disk are in `@dogsbay/site-check/node`.
|
|
7
|
+
*/
|
|
8
|
+
import { registerSeoRules } from "./rules/seo/index.js";
|
|
9
|
+
import { registerStructureRules } from "./rules/structure/index.js";
|
|
10
|
+
import { registerFrontmatterRules } from "./rules/frontmatter/index.js";
|
|
11
|
+
export * from "./types.js";
|
|
12
|
+
export { registerRule, listRules, getRule, filterRules, _clearRegistry } from "./registry.js";
|
|
13
|
+
export { runAudit, reportExitCode } from "./run.js";
|
|
14
|
+
export { formatJson } from "./output/json.js";
|
|
15
|
+
export { formatText } from "./output/text.js";
|
|
16
|
+
export { toReport } from "./run.js";
|
|
17
|
+
export { createAuditSession, slimPage } from "./session.js";
|
|
18
|
+
export { parseHtmlFile } from "./parse.js";
|
|
19
|
+
export { validateRefs } from "./lib/validate-refs.js";
|
|
20
|
+
export { compareRelease, releasePage, releaseBaseline, normalizeRemoved, signature, similarity, treeText, MOVED_SIMILARITY, MIN_SHINGLES } from "./lib/release.js";
|
|
21
|
+
export { formatReleaseText, releaseRedirects, markWritten, releaseOpen, releaseExitCode, formatWriteSummary } from "./output/release.js";
|
|
22
|
+
export { addRedirectsToConfig, redirectsBlock } from "./lib/config-redirects.js";
|
|
23
|
+
export { registerSeoRules, _resetSeoRegistration } from "./rules/seo/index.js";
|
|
24
|
+
export { registerStructureRules, _resetStructureRegistration } from "./rules/structure/index.js";
|
|
25
|
+
export { registerFrontmatterRules, _resetFrontmatterRegistration } from "./rules/frontmatter/index.js";
|
|
26
|
+
/** Register every shared rule (SEO, structure and frontmatter). Idempotent. */
|
|
27
|
+
export function registerSharedRules() {
|
|
28
|
+
registerSeoRules();
|
|
29
|
+
registerStructureRules();
|
|
30
|
+
registerFrontmatterRules();
|
|
31
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Add redirects to a `dogsbay.config.yml` as text, keeping its comments and
|
|
3
|
+
* layout: new `"/from": "/to"` lines go at the end of the top-level
|
|
4
|
+
* `redirects:` block, or in a new block at the end of the file. Both
|
|
5
|
+
* pipelines read that block the same way (site paths, no base path).
|
|
6
|
+
*
|
|
7
|
+
* A `redirects:` written in flow style (`redirects: { … }`) is not edited:
|
|
8
|
+
* the caller gets an error and prints the lines to paste instead.
|
|
9
|
+
*/
|
|
10
|
+
export interface ConfigRedirect {
|
|
11
|
+
/** Site paths without a leading slash or base path: `latest/guide/old`. */
|
|
12
|
+
from: string;
|
|
13
|
+
to: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function addRedirectsToConfig(text: string, redirects: ConfigRedirect[],
|
|
16
|
+
/** Sources the config already redirects, as written (`/latest/guide/old`); these are skipped. */
|
|
17
|
+
existing?: Iterable<string>): {
|
|
18
|
+
text: string;
|
|
19
|
+
added: ConfigRedirect[];
|
|
20
|
+
};
|
|
21
|
+
/** A block list of the same redirects, to print for pasting. */
|
|
22
|
+
export declare function redirectsBlock(redirects: ConfigRedirect[]): string;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export function addRedirectsToConfig(text, redirects,
|
|
2
|
+
/** Sources the config already redirects, as written (`/latest/guide/old`); these are skipped. */
|
|
3
|
+
existing = []) {
|
|
4
|
+
const have = new Set([...existing].map(norm));
|
|
5
|
+
const added = [];
|
|
6
|
+
for (const r of redirects) {
|
|
7
|
+
if (have.has(norm(r.from)))
|
|
8
|
+
continue;
|
|
9
|
+
have.add(norm(r.from));
|
|
10
|
+
added.push(r);
|
|
11
|
+
}
|
|
12
|
+
if (!added.length)
|
|
13
|
+
return { text, added };
|
|
14
|
+
// Written back with the file's own line ending.
|
|
15
|
+
const eol = text.includes("\r\n") ? "\r\n" : "\n";
|
|
16
|
+
const lines = text.split(/\r?\n/);
|
|
17
|
+
const key = /^(["']?)redirects\1\s*:/;
|
|
18
|
+
const at = lines.findIndex((l) => key.test(l));
|
|
19
|
+
const entries = added.map((r) => entry(r));
|
|
20
|
+
if (at < 0) {
|
|
21
|
+
const body = text === "" || /\r?\n$/.test(text) ? text : text + eol;
|
|
22
|
+
return { text: body + ["redirects:", ...entries.map((e) => " " + e)].join(eol) + eol, added };
|
|
23
|
+
}
|
|
24
|
+
const value = lines[at].replace(key, "").replace(/\s+#.*$/, "").trim();
|
|
25
|
+
// An empty value (`redirects:`, `~`, `null`, `{}`) becomes a block; any other value on the line is not edited.
|
|
26
|
+
if (value === "~" || value === "null" || value === "{}") {
|
|
27
|
+
lines.splice(at, 1, "redirects:", ...entries.map((e) => " " + e));
|
|
28
|
+
return { text: lines.join(eol), added };
|
|
29
|
+
}
|
|
30
|
+
if (value !== "") {
|
|
31
|
+
throw new Error("redirects: is not a block mapping (one entry per line), so it is not edited; add these lines to it by hand");
|
|
32
|
+
}
|
|
33
|
+
// The block runs to the next line that is not indented, blank or a comment.
|
|
34
|
+
let indent = " ";
|
|
35
|
+
let last = at;
|
|
36
|
+
for (let i = at + 1; i < lines.length; i++) {
|
|
37
|
+
const l = lines[i];
|
|
38
|
+
if (l.trim() === "" || /^\s+#/.test(l))
|
|
39
|
+
continue;
|
|
40
|
+
const m = /^(\s+)\S/.exec(l);
|
|
41
|
+
if (!m)
|
|
42
|
+
break;
|
|
43
|
+
if (last === at)
|
|
44
|
+
indent = m[1];
|
|
45
|
+
last = i;
|
|
46
|
+
}
|
|
47
|
+
lines.splice(last + 1, 0, ...entries.map((e) => indent + e));
|
|
48
|
+
return { text: lines.join(eol), added };
|
|
49
|
+
}
|
|
50
|
+
/** A block list of the same redirects, to print for pasting. */
|
|
51
|
+
export function redirectsBlock(redirects) {
|
|
52
|
+
return "redirects:\n" + redirects.map((r) => ` ${entry(r)}\n`).join("");
|
|
53
|
+
}
|
|
54
|
+
function entry(r) {
|
|
55
|
+
return `${quote("/" + r.from.replace(/^\/+/, ""))}: ${quote("/" + r.to.replace(/^\/+/, ""))}`;
|
|
56
|
+
}
|
|
57
|
+
function quote(s) {
|
|
58
|
+
return /^[\w/.\-~]+$/.test(s) ? s : JSON.stringify(s);
|
|
59
|
+
}
|
|
60
|
+
function norm(p) {
|
|
61
|
+
return p.replace(/^\/+|\/+$/g, "").replace(/\.html$/, "");
|
|
62
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small helpers shared by the streamed session and the rules.
|
|
3
|
+
*/
|
|
4
|
+
import type { ExportPage } from "@dogsbay/types";
|
|
5
|
+
/** The title a page is shown with: front matter first, then the one the build took from the first H1. */
|
|
6
|
+
export declare function titleOf(page: ExportPage): string;
|
|
7
|
+
/**
|
|
8
|
+
* A standalone copy of `s`. V8 keeps a slice or a regex match as a view on
|
|
9
|
+
* its parent string, so a kept href would otherwise pin its whole page.
|
|
10
|
+
*/
|
|
11
|
+
export declare function flat(s: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* One key per page, whatever form names it: a file path (`guide/x.html`,
|
|
14
|
+
* `guide/x/index.html`) or a URL (`https://site/base/guide/x/`,
|
|
15
|
+
* `/guide/x.html`, `…/guide/x` with pretty links). Origin, the site URL's
|
|
16
|
+
* path, query and hash go; `index.html`, `.html` and slashes at the ends are
|
|
17
|
+
* dropped, so the forms meet at `guide/x`.
|
|
18
|
+
*/
|
|
19
|
+
export declare function pageKey(pathOrUrl: string, urlBase?: string): string;
|
package/dist/lib/page.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** The title a page is shown with: front matter first, then the one the build took from the first H1. */
|
|
2
|
+
export function titleOf(page) {
|
|
3
|
+
const fm = page.frontmatter?.["title"];
|
|
4
|
+
return (typeof fm === "string" && fm.trim() ? fm : page.title ?? "").trim();
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* A standalone copy of `s`. V8 keeps a slice or a regex match as a view on
|
|
8
|
+
* its parent string, so a kept href would otherwise pin its whole page.
|
|
9
|
+
*/
|
|
10
|
+
export function flat(s) {
|
|
11
|
+
return JSON.parse(JSON.stringify(s));
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* One key per page, whatever form names it: a file path (`guide/x.html`,
|
|
15
|
+
* `guide/x/index.html`) or a URL (`https://site/base/guide/x/`,
|
|
16
|
+
* `/guide/x.html`, `…/guide/x` with pretty links). Origin, the site URL's
|
|
17
|
+
* path, query and hash go; `index.html`, `.html` and slashes at the ends are
|
|
18
|
+
* dropped, so the forms meet at `guide/x`.
|
|
19
|
+
*/
|
|
20
|
+
export function pageKey(pathOrUrl, urlBase = "") {
|
|
21
|
+
let p = pathOrUrl.replace(/[?#].*$/, "");
|
|
22
|
+
const m = /^[a-z][a-z0-9+.-]*:\/\/[^/]*(.*)$/i.exec(p) ?? /^\/\/[^/]*(.*)$/.exec(p);
|
|
23
|
+
if (m)
|
|
24
|
+
p = m[1];
|
|
25
|
+
p = p.replace(/^\/+/, "");
|
|
26
|
+
const base = urlBase.replace(/^\/+|\/+$/g, "");
|
|
27
|
+
if (m && base && (p === base || p.startsWith(base + "/")))
|
|
28
|
+
p = p.slice(base.length);
|
|
29
|
+
return p
|
|
30
|
+
.replace(/^\/+|\/+$/g, "")
|
|
31
|
+
.replace(/(^|\/)index\.html$/, "")
|
|
32
|
+
.replace(/\.html$/, "")
|
|
33
|
+
.replace(/\/+$/, "");
|
|
34
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a release moved or removed: the pages of an older version that the
|
|
3
|
+
* default version no longer has at the same path, each classified as
|
|
4
|
+
* redirected, recorded as removed, probably moved (with the evidence) or
|
|
5
|
+
* gone. `dogsbay site release-diff` and `dogsbay-ssg release-diff` print it;
|
|
6
|
+
* the `structure/release-moves` rule reports the open ones.
|
|
7
|
+
*
|
|
8
|
+
* Pipeline-neutral: each pipeline passes its pages as `ReleasePage`s with
|
|
9
|
+
* paths inside their version (`guide/install`, `index` for the home page)
|
|
10
|
+
* and the paths its redirects cover. Worker-safe; the git rename signal is
|
|
11
|
+
* read by the Node adapter (`gitRenames` in `@dogsbay/site-check/node`) and
|
|
12
|
+
* passed in. See dogsbay-ssg plans/canonical-versions.md, step 3.
|
|
13
|
+
*/
|
|
14
|
+
import type { ExportPage, TreeNode } from "@dogsbay/types";
|
|
15
|
+
/** One page of a version, reduced to what matching needs (small: kept for two versions only). */
|
|
16
|
+
export interface ReleasePage {
|
|
17
|
+
/** Path inside the version: `guide/install`; `index` for the home page. */
|
|
18
|
+
path: string;
|
|
19
|
+
title: string;
|
|
20
|
+
/** Front matter `id:` (a stable id, such as a DITA topic id), when the page has one. */
|
|
21
|
+
id?: string;
|
|
22
|
+
/** The page's source file, relative to the project (reported by `structure/release-moves`). */
|
|
23
|
+
source?: string;
|
|
24
|
+
/** The page comes from a folder's index file (`admin/index.md`), so it is served as a folder: its redirect source ends in `/`. */
|
|
25
|
+
index?: boolean;
|
|
26
|
+
/** MinHash of the page's word shingles; see `similarity`. */
|
|
27
|
+
signature: number[];
|
|
28
|
+
/** How many shingles the signature was taken over; a short page's text is no evidence of a move. */
|
|
29
|
+
shingles: number;
|
|
30
|
+
}
|
|
31
|
+
export type ReleaseStatus = "redirected" | "removed-ok" | "moved" | "candidate" | "removed";
|
|
32
|
+
export interface ReleaseEntry {
|
|
33
|
+
/** The page's path in the older version. */
|
|
34
|
+
path: string;
|
|
35
|
+
status: ReleaseStatus;
|
|
36
|
+
/** Where it went (`moved`, `candidate`) or where its redirect points (`redirected`). */
|
|
37
|
+
to?: string;
|
|
38
|
+
/** Why it is thought to have moved: `git rename`, `same id`, `same text, 84%`, `same title`, `same file name`. */
|
|
39
|
+
evidence?: string[];
|
|
40
|
+
/** The older page's source file, relative to the project, when the pipeline knows it. */
|
|
41
|
+
source?: string;
|
|
42
|
+
/** The older page was a folder's index file; see `ReleasePage.index`. */
|
|
43
|
+
index?: true;
|
|
44
|
+
}
|
|
45
|
+
export interface CompareReleaseInput {
|
|
46
|
+
/** The older version's pages. */
|
|
47
|
+
from: ReleasePage[];
|
|
48
|
+
/** The default version's pages. */
|
|
49
|
+
to: ReleasePage[];
|
|
50
|
+
/**
|
|
51
|
+
* Redirects in the default version, by source path inside the version. The
|
|
52
|
+
* target is a path inside the version, a site path with a leading slash
|
|
53
|
+
* when it is outside the version (`/other/x`), or a URL.
|
|
54
|
+
*/
|
|
55
|
+
redirects?: Map<string, string>;
|
|
56
|
+
/** Paths recorded as removed on purpose (`versions[].removed` of the default version). */
|
|
57
|
+
removed?: string[];
|
|
58
|
+
/** Renames in the default version's source since the older release, from git (old path → new path). */
|
|
59
|
+
renames?: Map<string, string>;
|
|
60
|
+
}
|
|
61
|
+
/** Estimated text overlap from which a new page counts as the same page, moved. */
|
|
62
|
+
export declare const MOVED_SIMILARITY = 0.6;
|
|
63
|
+
/** Shingles both pages need for their text to count ("Coming soon" stubs all look alike). About a dozen words. */
|
|
64
|
+
export declare const MIN_SHINGLES = 8;
|
|
65
|
+
/**
|
|
66
|
+
* Classify every page of `from` that `to` does not have at the same path.
|
|
67
|
+
* Unchanged pages are left out. Order: `from`'s.
|
|
68
|
+
*/
|
|
69
|
+
export declare function compareRelease(input: CompareReleaseInput): ReleaseEntry[];
|
|
70
|
+
/**
|
|
71
|
+
* A `removed:` entry as a page path: no slashes at the ends, no `.md`,
|
|
72
|
+
* `.markdown` or `.mdx`, `x/index` → `x`, and the home page `index`.
|
|
73
|
+
*/
|
|
74
|
+
export declare function normalizeRemoved(raw: string): string;
|
|
75
|
+
/**
|
|
76
|
+
* A page as matching sees it. `path` is its path inside its version;
|
|
77
|
+
* `source` its source file relative to the project; `index` whether that
|
|
78
|
+
* file is a folder's index (other than the home page).
|
|
79
|
+
*/
|
|
80
|
+
export declare function releasePage(page: ExportPage, path: string, o?: {
|
|
81
|
+
source?: string;
|
|
82
|
+
index?: boolean;
|
|
83
|
+
}): ReleasePage;
|
|
84
|
+
/**
|
|
85
|
+
* MinHash over the text's 5-word shingles (lowercased words; a text shorter
|
|
86
|
+
* than one shingle is one shingle): for each of 64 seeded hashes, the
|
|
87
|
+
* smallest value over the shingles. Two signatures agree in a slot with the
|
|
88
|
+
* probability that the two shingle sets overlap (their Jaccard index).
|
|
89
|
+
*/
|
|
90
|
+
export declare function signature(text: string): number[];
|
|
91
|
+
/** Estimated Jaccard index of two pages' shingle sets: the share of MinHash slots that agree. Empty pages match nothing. */
|
|
92
|
+
export declare function similarity(a: number[], b: number[]): number;
|
|
93
|
+
/** A page's text: inline text, HTML without its tags, and code, in document order. */
|
|
94
|
+
export declare function treeText(nodes: TreeNode[]): string;
|
|
95
|
+
/**
|
|
96
|
+
* The version a release is compared against: the first version after the
|
|
97
|
+
* default one in `versions` order (newest first) that is neither a
|
|
98
|
+
* pre-release nor `hidden` (the numbered twin of a `latest` alias, the same
|
|
99
|
+
* pages); with the snapshot routine, the previous release as published.
|
|
100
|
+
* `from` names one instead.
|
|
101
|
+
*/
|
|
102
|
+
export declare function releaseBaseline(versions: {
|
|
103
|
+
id: string;
|
|
104
|
+
prerelease?: boolean;
|
|
105
|
+
hidden?: boolean;
|
|
106
|
+
}[], defaultVersion: string | undefined, from?: string): string | undefined;
|
|
107
|
+
/** One comparison, as a pipeline hands it to the audit: the versions, the language, and the entries. */
|
|
108
|
+
export interface ReleaseComparison {
|
|
109
|
+
/** The older version's id. */
|
|
110
|
+
from: string;
|
|
111
|
+
/** The default version's id. */
|
|
112
|
+
to: string;
|
|
113
|
+
locale?: string;
|
|
114
|
+
/** Site path prefix of each version's pages (`v3`, `en/v3`; empty when unprefixed). */
|
|
115
|
+
fromPrefix: string;
|
|
116
|
+
toPrefix: string;
|
|
117
|
+
entries: ReleaseEntry[];
|
|
118
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { flat, titleOf } from "./page.js";
|
|
2
|
+
/** Estimated text overlap from which a new page counts as the same page, moved. */
|
|
3
|
+
export const MOVED_SIMILARITY = 0.6;
|
|
4
|
+
/** Shingles both pages need for their text to count ("Coming soon" stubs all look alike). About a dozen words. */
|
|
5
|
+
export const MIN_SHINGLES = 8;
|
|
6
|
+
/**
|
|
7
|
+
* Classify every page of `from` that `to` does not have at the same path.
|
|
8
|
+
* Unchanged pages are left out. Order: `from`'s.
|
|
9
|
+
*/
|
|
10
|
+
export function compareRelease(input) {
|
|
11
|
+
const toPaths = new Set(input.to.map((p) => p.path));
|
|
12
|
+
const fromPaths = new Set(input.from.map((p) => p.path));
|
|
13
|
+
const removed = new Set((input.removed ?? []).map(normalizeRemoved));
|
|
14
|
+
// A move lands on a page the older version did not have. Sorted, so a tie goes the same way in every pipeline.
|
|
15
|
+
const fresh = input.to.filter((p) => !fromPaths.has(p.path)).sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
16
|
+
const entries = [];
|
|
17
|
+
for (const page of input.from) {
|
|
18
|
+
if (toPaths.has(page.path))
|
|
19
|
+
continue;
|
|
20
|
+
const about = { ...(page.source !== undefined ? { source: page.source } : {}), ...(page.index ? { index: true } : {}) };
|
|
21
|
+
const redirect = input.redirects?.get(page.path);
|
|
22
|
+
if (redirect !== undefined) {
|
|
23
|
+
entries.push({ path: page.path, status: "redirected", to: redirect, ...about });
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
if (removed.has(page.path)) {
|
|
27
|
+
entries.push({ path: page.path, status: "removed-ok", ...about });
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
entries.push({ ...match(page, fresh, toPaths, input.renames), ...about });
|
|
31
|
+
}
|
|
32
|
+
return entries;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* A `removed:` entry as a page path: no slashes at the ends, no `.md`,
|
|
36
|
+
* `.markdown` or `.mdx`, `x/index` → `x`, and the home page `index`.
|
|
37
|
+
*/
|
|
38
|
+
export function normalizeRemoved(raw) {
|
|
39
|
+
const p = raw
|
|
40
|
+
.trim()
|
|
41
|
+
.replace(/^\/+|\/+$/g, "")
|
|
42
|
+
.replace(/\.(md|markdown|mdx)$/i, "")
|
|
43
|
+
.replace(/(^|\/)index$/, "");
|
|
44
|
+
return p || "index";
|
|
45
|
+
}
|
|
46
|
+
function match(page, fresh, toPaths, renames) {
|
|
47
|
+
const renamed = renames?.get(page.path);
|
|
48
|
+
if (renamed !== undefined && toPaths.has(renamed)) {
|
|
49
|
+
return { path: page.path, status: "moved", to: renamed, evidence: ["git rename"] };
|
|
50
|
+
}
|
|
51
|
+
if (page.id) {
|
|
52
|
+
const same = fresh.find((p) => p.id === page.id);
|
|
53
|
+
if (same)
|
|
54
|
+
return { path: page.path, status: "moved", to: same.path, evidence: ["same id"] };
|
|
55
|
+
}
|
|
56
|
+
let best;
|
|
57
|
+
let bestScore = 0;
|
|
58
|
+
for (const p of page.shingles >= MIN_SHINGLES ? fresh : []) {
|
|
59
|
+
if (p.shingles < MIN_SHINGLES)
|
|
60
|
+
continue;
|
|
61
|
+
const s = similarity(page.signature, p.signature);
|
|
62
|
+
if (s > bestScore) {
|
|
63
|
+
best = p;
|
|
64
|
+
bestScore = s;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (best && bestScore >= MOVED_SIMILARITY) {
|
|
68
|
+
return { path: page.path, status: "moved", to: best.path, evidence: [`same text, ${Math.round(bestScore * 100)}%`] };
|
|
69
|
+
}
|
|
70
|
+
const title = page.title.toLowerCase();
|
|
71
|
+
const byTitle = title ? fresh.find((p) => p.title.toLowerCase() === title) : undefined;
|
|
72
|
+
if (byTitle)
|
|
73
|
+
return { path: page.path, status: "candidate", to: byTitle.path, evidence: ["same title"] };
|
|
74
|
+
const name = baseName(page.path);
|
|
75
|
+
const byName = fresh.find((p) => baseName(p.path) === name);
|
|
76
|
+
if (byName)
|
|
77
|
+
return { path: page.path, status: "candidate", to: byName.path, evidence: ["same file name"] };
|
|
78
|
+
return { path: page.path, status: "removed" };
|
|
79
|
+
}
|
|
80
|
+
/** The last segment that names the page: `guide/install` → `install`, `guide/index` → `guide`. */
|
|
81
|
+
function baseName(path) {
|
|
82
|
+
const parts = path.split("/").filter((s) => s !== "index");
|
|
83
|
+
return parts[parts.length - 1] ?? "index";
|
|
84
|
+
}
|
|
85
|
+
// ── Page fingerprints ────────────────────────────────────────────
|
|
86
|
+
const HASHES = 64;
|
|
87
|
+
const SHINGLE = 5;
|
|
88
|
+
/**
|
|
89
|
+
* A page as matching sees it. `path` is its path inside its version;
|
|
90
|
+
* `source` its source file relative to the project; `index` whether that
|
|
91
|
+
* file is a folder's index (other than the home page).
|
|
92
|
+
*/
|
|
93
|
+
export function releasePage(page, path, o = {}) {
|
|
94
|
+
const id = page.frontmatter?.["id"];
|
|
95
|
+
const text = treeText(page.tree);
|
|
96
|
+
return {
|
|
97
|
+
path: flat(path),
|
|
98
|
+
title: flat(titleOf(page)),
|
|
99
|
+
...(typeof id === "string" && id.trim() ? { id: flat(id.trim()) } : {}),
|
|
100
|
+
...(o.source !== undefined ? { source: flat(o.source) } : {}),
|
|
101
|
+
...(o.index && path !== "index" ? { index: true } : {}),
|
|
102
|
+
signature: signature(text),
|
|
103
|
+
shingles: shingleCount(text),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
function shingleCount(text) {
|
|
107
|
+
const words = text.match(/[\p{L}\p{N}]+/gu)?.length ?? 0;
|
|
108
|
+
return words ? Math.max(1, words - SHINGLE + 1) : 0;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* MinHash over the text's 5-word shingles (lowercased words; a text shorter
|
|
112
|
+
* than one shingle is one shingle): for each of 64 seeded hashes, the
|
|
113
|
+
* smallest value over the shingles. Two signatures agree in a slot with the
|
|
114
|
+
* probability that the two shingle sets overlap (their Jaccard index).
|
|
115
|
+
*/
|
|
116
|
+
export function signature(text) {
|
|
117
|
+
const words = text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
118
|
+
const mins = new Array(HASHES).fill(0xffffffff);
|
|
119
|
+
if (!words.length)
|
|
120
|
+
return mins;
|
|
121
|
+
const count = Math.max(1, words.length - SHINGLE + 1);
|
|
122
|
+
for (let i = 0; i < count; i++) {
|
|
123
|
+
const h = fnv1a(words.slice(i, i + SHINGLE).join(" "));
|
|
124
|
+
for (let k = 0; k < HASHES; k++) {
|
|
125
|
+
const v = mix(h, k);
|
|
126
|
+
if (v < mins[k])
|
|
127
|
+
mins[k] = v;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return mins;
|
|
131
|
+
}
|
|
132
|
+
/** Estimated Jaccard index of two pages' shingle sets: the share of MinHash slots that agree. Empty pages match nothing. */
|
|
133
|
+
export function similarity(a, b) {
|
|
134
|
+
if (a.length !== b.length || !a.length || a[0] === 0xffffffff || b[0] === 0xffffffff)
|
|
135
|
+
return 0;
|
|
136
|
+
let same = 0;
|
|
137
|
+
for (let i = 0; i < a.length; i++)
|
|
138
|
+
if (a[i] === b[i])
|
|
139
|
+
same++;
|
|
140
|
+
return same / a.length;
|
|
141
|
+
}
|
|
142
|
+
function fnv1a(s) {
|
|
143
|
+
let h = 0x811c9dc5;
|
|
144
|
+
for (let i = 0; i < s.length; i++) {
|
|
145
|
+
h ^= s.charCodeAt(i);
|
|
146
|
+
h = Math.imul(h, 0x01000193);
|
|
147
|
+
}
|
|
148
|
+
return h >>> 0;
|
|
149
|
+
}
|
|
150
|
+
/** The k-th hash of `h` (a murmur3 finaliser over h and a per-k seed). */
|
|
151
|
+
function mix(h, k) {
|
|
152
|
+
let x = (h ^ Math.imul(k + 1, 0x9e3779b9)) >>> 0;
|
|
153
|
+
x ^= x >>> 16;
|
|
154
|
+
x = Math.imul(x, 0x85ebca6b);
|
|
155
|
+
x ^= x >>> 13;
|
|
156
|
+
x = Math.imul(x, 0xc2b2ae35);
|
|
157
|
+
x ^= x >>> 16;
|
|
158
|
+
return x >>> 0;
|
|
159
|
+
}
|
|
160
|
+
/** A page's text: inline text, HTML without its tags, and code, in document order. */
|
|
161
|
+
export function treeText(nodes) {
|
|
162
|
+
const out = [];
|
|
163
|
+
const walk = (list) => {
|
|
164
|
+
for (const node of list) {
|
|
165
|
+
if (node.inline)
|
|
166
|
+
out.push(inlineText(node.inline));
|
|
167
|
+
if (node.html)
|
|
168
|
+
out.push(node.html.replace(/<[^>]+>/g, " "));
|
|
169
|
+
if (typeof node.props?.code === "string")
|
|
170
|
+
out.push(node.props.code);
|
|
171
|
+
if (node.children)
|
|
172
|
+
walk(node.children);
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
walk(nodes);
|
|
176
|
+
return out.join(" ");
|
|
177
|
+
}
|
|
178
|
+
function inlineText(nodes) {
|
|
179
|
+
let s = "";
|
|
180
|
+
for (const n of nodes) {
|
|
181
|
+
if (typeof n.text === "string")
|
|
182
|
+
s += n.text;
|
|
183
|
+
else if (typeof n.value === "string")
|
|
184
|
+
s += n.value;
|
|
185
|
+
if (Array.isArray(n.children))
|
|
186
|
+
s += inlineText(n.children);
|
|
187
|
+
s += " ";
|
|
188
|
+
}
|
|
189
|
+
return s;
|
|
190
|
+
}
|
|
191
|
+
// ── Which versions to compare ────────────────────────────────────
|
|
192
|
+
/**
|
|
193
|
+
* The version a release is compared against: the first version after the
|
|
194
|
+
* default one in `versions` order (newest first) that is neither a
|
|
195
|
+
* pre-release nor `hidden` (the numbered twin of a `latest` alias, the same
|
|
196
|
+
* pages); with the snapshot routine, the previous release as published.
|
|
197
|
+
* `from` names one instead.
|
|
198
|
+
*/
|
|
199
|
+
export function releaseBaseline(versions, defaultVersion, from) {
|
|
200
|
+
if (from !== undefined)
|
|
201
|
+
return versions.some((v) => v.id === from) && from !== defaultVersion ? from : undefined;
|
|
202
|
+
const i = versions.findIndex((v) => v.id === defaultVersion);
|
|
203
|
+
if (i < 0)
|
|
204
|
+
return undefined;
|
|
205
|
+
return versions.slice(i + 1).find((v) => !v.prerelease && !v.hidden)?.id;
|
|
206
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure validator for internal page links + asset references in
|
|
3
|
+
* imported content. Walks every `ExportPage`'s tree, collects
|
|
4
|
+
* every internal `<a href>` and `<img src>`, set-checks against
|
|
5
|
+
* the page slug set + the asset file set. A relative reference is
|
|
6
|
+
* resolved against the page's own file (`links.ts`), as the build
|
|
7
|
+
* resolves it.
|
|
8
|
+
*
|
|
9
|
+
* Single source of truth shared between two surfaces:
|
|
10
|
+
* - `structure/internal-links` + `structure/asset-refs` audit
|
|
11
|
+
* rules (warning-or-error issues from `dogsbay site check`)
|
|
12
|
+
* - `dogsbay site build --strict` (throws on first miss)
|
|
13
|
+
*
|
|
14
|
+
* See plans/audit-internal-refs.md.
|
|
15
|
+
*/
|
|
16
|
+
import type { ExportPage } from "@dogsbay/types";
|
|
17
|
+
/** One miss found by the validator. */
|
|
18
|
+
export interface RefIssue {
|
|
19
|
+
/** Slug of the page that contains the broken reference. */
|
|
20
|
+
pageSlug: string;
|
|
21
|
+
/** Raw href / src as written by the author. */
|
|
22
|
+
href: string;
|
|
23
|
+
/** Whether the validator was looking for a page or an asset. */
|
|
24
|
+
resolvedAgainst: "page" | "asset";
|
|
25
|
+
}
|
|
26
|
+
export interface ValidateRefsOptions {
|
|
27
|
+
/** Site basePath (e.g. "/docs", "/" for host-root, "" for root). */
|
|
28
|
+
basePath: string;
|
|
29
|
+
/**
|
|
30
|
+
* Absolute paths under the public asset tree, leading slash,
|
|
31
|
+
* matching how authors write them. e.g. `"/_assets/foo.png"`.
|
|
32
|
+
*/
|
|
33
|
+
knownAssets: Set<string>;
|
|
34
|
+
}
|
|
35
|
+
export interface ValidateRefsResult {
|
|
36
|
+
linkMisses: RefIssue[];
|
|
37
|
+
assetMisses: RefIssue[];
|
|
38
|
+
}
|
|
39
|
+
export declare function validateRefs(pages: ExportPage[], options: ValidateRefsOptions): ValidateRefsResult;
|