@d-zero/page-cluster 0.3.0 → 0.4.0

package/dist/resolve-page-cluster-keys.js

@@ -1,4 +1,5 @@
 import { autoCutThreshold } from './auto-cut-threshold.js';
+import { buildPageLandmarkReport, } from './build-page-landmark-report.js';
 import { capContentDepth } from './cap-content-depth.js';
 import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js';
 import { extractLandmarks } from './extract-landmarks.js';
@@ -8,6 +9,7 @@ import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js';
 import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js';
 import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js';
 import { removeContentBlocks } from './remove-content-blocks.js';
+import { shellQuorum } from './shell-quorum.js';
 import { stageAPerBlock } from './stage-a-per-block.js';
 import { tokenize } from './tokenize.js';
 /**
@@ -162,8 +164,9 @@ function assignPageToNearestCluster(html, assignment, excludeLandmarks, contentB
  */
 export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
     const pageCount = landmarks.length;
-    if (pageCount === 0)
-        return { localSignatures: new Set(), localTokensByPage: [] };
+    if (pageCount === 0) {
+        return { localSignatures: new Set(), localTokensByPage: [], perPageInstances: [] };
+    }
     const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions);
     // Corpus-wide histogram: signature → { count, tokens }. tokens is the
     // token set of any one occurrence of the signature (all occurrences are
@@ -184,6 +187,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const frequencies = [];
@@ -202,6 +206,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         return {
             localSignatures: new Set(),
             localTokensByPage: landmarks.map(() => new Set()),
+            perPageInstances,
         };
     }
     const localTokensByPage = perPageInstances.map((instances) => {
@@ -214,7 +219,7 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
         }
         return out;
     });
-    return { localSignatures, localTokensByPage };
+    return { localSignatures, localTokensByPage, perPageInstances };
 }
 /**
  * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens
@@ -232,6 +237,39 @@ export function computeLocalChromeArtifacts(landmarks, tokenizeOptions) {
 export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
     return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage];
 }
+/**
+ * Builds every page's {@link PageLandmarkReport} for the `includeLandmarkPositions`
+ * result path. Groups pages by their *final* cluster key (post–Stage-B), runs
+ * {@link ./shell-quorum.js | shellQuorum} once per cluster over the pooled
+ * member `PerPageLandmarkInstance`s (the same "shell tokens" concept Stage B
+ * itself uses for L2 shell corroboration, just recomputed at the final-
+ * cluster granularity rather than the pre-merge unit granularity), then
+ * classifies each member page's own landmark instances against that
+ * cluster-level shell.
+ * @param finalKeys
+ * @param landmarks
+ * @param perPageInstances
+ */
+function buildLandmarkReportsByCluster(finalKeys, landmarks, perPageInstances) {
+    const indicesByKey = new Map();
+    for (const [i, key] of finalKeys.entries()) {
+        const indices = indicesByKey.get(key);
+        if (indices) {
+            indices.push(i);
+        }
+        else {
+            indicesByKey.set(key, [i]);
+        }
+    }
+    const reports = Array.from({ length: finalKeys.length });
+    for (const indices of indicesByKey.values()) {
+        const shellTokens = shellQuorum(indices.map((i) => perPageInstances[i]));
+        for (const i of indices) {
+            reports[i] = buildPageLandmarkReport(landmarks[i], shellTokens);
+        }
+    }
+    return reports;
+}
 /**
  * Corpus size at or below which the async factory-based
  * `resolvePageClusterKeys` reads the entire input into an array and delegates
@@ -254,6 +292,21 @@ export function computeLocalLandmarkTokens(landmarks, tokenizeOptions) {
  * anything above 20,000 is routed to streaming.
  */
 export const CORPUS_INLINE_THRESHOLD = 20_000;
+/**
+ * Throws when `includeLandmarkPositions` is combined with a corpus over
+ * `threshold` pages (the streaming path — see `includeLandmarkPositions`'s
+ * own JSDoc for why it has no sample-based equivalent there). Split out from
+ * its call site so tests can exercise the boundary with a small injected
+ * `threshold` instead of constructing a 20,001-page fixture to cross the
+ * real {@link CORPUS_INLINE_THRESHOLD}.
+ * @param pageCount
+ * @param threshold
+ */
+export function assertLandmarkPositionsSupportedForPageCount(pageCount, threshold) {
+    if (pageCount > threshold) {
+        throw new RangeError(`resolvePageClusterKeys: includeLandmarkPositions is not supported for corpora larger than ${threshold} pages (the streaming path) — this corpus has ${pageCount}`);
+    }
+}
 /**
  * Reservoir-sample size per block on the streaming path. Blocks larger than
  * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages,
@@ -289,23 +342,6 @@ export const CORPUS_INLINE_THRESHOLD = 20_000;
  * {@link CORPUS_INLINE_THRESHOLD} — sampling is streaming-mode only.
  */
 export const BLOCK_SAMPLE_SIZE = 100;
-/**
- * Preserves the previous synchronous, array-in / array-out API of
- * `resolvePageClusterKeys` under a new name so the factory-based async
- * export can take the primary name while callers that already had a
- * materialized page array (spec tests, the in-repo dogfood harness,
- * downstream code that hasn't switched to streaming yet) retain the
- * exact same behavior.
- *
- * Semantics: identical to the pre-refactor `resolvePageClusterKeys`.
- * Corpus-wide chrome discovery, Stage B across every page, no memory
- * bound — meant to be called on inputs already known to fit in memory.
- * The async factory-based export delegates here whenever
- * `pages.length ≤ CORPUS_INLINE_THRESHOLD`, guaranteeing existing corpora
- * hit exactly this code path.
- * @param pages
- * @param options
- */
 export function resolvePageClusterKeysInMemory(pages, options) {
     const excludeLandmarks = options?.excludeLandmarks ?? true;
     const similarityThreshold = options?.similarityThreshold ?? 0.8;
@@ -316,8 +352,11 @@ export function resolvePageClusterKeysInMemory(pages, options) {
     // corroboration regardless of `excludeLandmarks`, and `remainderHtml` is
     // needed whenever `excludeLandmarks` is true.
     const landmarks = pages.map((page) => extractLandmarks(page.html));
-    // Corpus-level chrome discovery
-    const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options);
+    // Corpus-level chrome discovery. `perPageInstances` is only consumed
+    // below when `includeLandmarkPositions` is set — computed unconditionally
+    // anyway since `computeLocalChromeArtifacts` already builds it internally
+    // for chrome discovery, so exposing it here costs nothing extra.
+    const { localTokensByPage: localLandmarkTokensByPage, perPageInstances } = computeLocalChromeArtifacts(landmarks, options);
     const contentBlockAttribute = options?.contentBlockAttribute;
     const preparedHtml = pages.map((page, index) => {
         const landmarksExcised = excludeLandmarks
@@ -363,7 +402,10 @@ export function resolvePageClusterKeysInMemory(pages, options) {
             finalKeys[i] = rootKey;
         }
     }
-    return finalKeys;
+    if (!options?.includeLandmarkPositions)
+        return finalKeys;
+    const reports = buildLandmarkReportsByCluster(finalKeys, landmarks, perPageInstances);
+    return finalKeys.map((clusterKey, i) => ({ clusterKey, landmarks: reports[i] }));
 }
 /**
  * Async twin of {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}
@@ -453,48 +495,6 @@ async function resolveSmallCorpusWithProgress(pages, onProgress, options) {
     }
     return finalKeys;
 }
-/**
- * Streaming, memory-bounded version of `resolvePageClusterKeysInMemory`.
- *
- * ## Behavior gate
- *
- * - `pageCount ≤ CORPUS_INLINE_THRESHOLD` — reads the whole factory into an
- *   array, delegates to `resolvePageClusterKeysInMemory`. Same corpus-wide
- *   chrome discovery, same Stage B across every page. All previously
- *   validated corpora (302 / 1,416 / 8,936 / 89 pages) hit this path.
- * - `pageCount > CORPUS_INLINE_THRESHOLD` — streaming path: reads the
- *   factory twice (once for blocking signals, once for HTML processing),
- *   dispatches HTML per block, runs Stage A per block, accumulates
- *   cross-block units, then runs Stage B across all accumulated units. Peak
- *   memory ≈ largest single block, not the whole corpus.
- *
- * ## Semantic differences in streaming mode
- *
- * - **Chrome discovery is per-block, not corpus-wide.** In the in-memory
- *   path, {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens}
- *   runs on all pages at once. In streaming mode the entire corpus cannot
- *   be held at once, so chrome discovery runs per block. A landmark
- *   signature that is rare corpus-wide but common within one block will
- *   be treated as global chrome in streaming mode, whereas the in-memory
- *   mode would treat it as local. This trade-off is why the threshold
- *   above is set generously — every real corpus historically validated
- *   here stays on the in-memory path.
- * @param pages
- * @param options
- * @example
- * ```ts
- * // JSONL file source — factory can be re-invoked to re-open the file.
- * import { createReadStream } from 'node:fs';
- * import readline from 'node:readline';
- *
- * const keys = await resolvePageClusterKeys(() => {
- *   const lines = readline.createInterface({ input: createReadStream('pages.jsonl') });
- *   return (async function* () {
- *     for await (const line of lines) yield JSON.parse(line);
- *   })();
- * });
- * ```
- */
 export async function resolvePageClusterKeys(pages, options) {
     const onProgress = options?.onProgress;
     // Pass 0: HTML-free — collect blocking signals (paths, stylesheetHrefs,
@@ -530,13 +530,22 @@ export async function resolvePageClusterKeys(pages, options) {
         // into per-block progress, so delegate to the untouched sync path —
         // keeping behavior byte-for-byte identical (and yield-overhead-free)
         // to how library-only consumers experienced this before the CLI
-        // progress work landed.
-        if (onProgress === undefined) {
+        // progress work landed. `includeLandmarkPositions` always routes here
+        // too (see its own JSDoc): `resolveSmallCorpusWithProgress` has no
+        // landmark-report support, and duplicating that logic into the
+        // progress-emitting path for a reporting feature that has nothing to
+        // do with progress observability isn't worth the added surface.
+        if (onProgress === undefined || options?.includeLandmarkPositions) {
             return resolvePageClusterKeysInMemory(fullPages, options);
         }
         return resolveSmallCorpusWithProgress(fullPages, onProgress, options);
     }
-    // Large corpus: streaming path.
+    // Large corpus: streaming path. includeLandmarkPositions has no sample-
+    // based equivalent (see its own JSDoc) — fail fast rather than silently
+    // ignoring the option or returning a semantically-wrong report.
+    if (options?.includeLandmarkPositions) {
+        assertLandmarkPositionsSupportedForPageCount(blockingSignals.length, CORPUS_INLINE_THRESHOLD);
+    }
     const excludeLandmarks = options?.excludeLandmarks ?? true;
     const similarityThreshold = options?.similarityThreshold ?? 0.8;
     if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) {
@@ -728,22 +737,6 @@ export async function resolvePageClusterKeys(pages, options) {
     }
     return finalKeys;
 }
-/**
- * Convenience wrapper that runs {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}
- * on a materialized array. Preserves the pre-refactor sync API for callers
- * that already have all pages in memory, while flowing through the same
- * async driver so behavior stays consistent across the two entry points.
- * @param pages
- * @param options
- * @example
- * ```ts
- * const keys = await resolvePageClusterKeysFromArray([
- *   { paths: ['news', '1'], stylesheetHrefs: [], html: '<body><article>one</article></body>' },
- *   { paths: ['news', '2'], stylesheetHrefs: [], html: '<body><article>two</article></body>' },
- *   { paths: ['about'], stylesheetHrefs: [], html: '<body><section>about</section></body>' },
- * ]);
- * ```
- */
 export function resolvePageClusterKeysFromArray(pages, options) {
     return resolvePageClusterKeys(() => pages, options);
 }

package/dist/shell-quorum.d.ts

@@ -0,0 +1,70 @@
+import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js';
+/**
+ * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
+ * frequency histogram of every landmark instance's tokens. This is the same
+ * max-gap primitive used for Stage A merge-height cutoffs, applied
+ * recursively at the landmark-token layer.
+ *
+ * ## Why per-token and not per-signature
+ *
+ * An earlier iteration ran the histogram at the level of full landmark-
+ * instance signatures (canonicalized token sets). That failed on a real,
+ * common pattern: a shared site chrome whose markup carries a per-page
+ * distinguishing element (a breadcrumb, a page-title element with a page-
+ * specific class, a "current" state). All pages have most of the same
+ * tokens, but every page's full signature is distinct because tokens embed
+ * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
+ * autoCutThreshold on the flat distribution returned the clamp, and the
+ * shell collapsed to empty even though every page shared the core header
+ * skeleton. Per-token counting handles the same case correctly — the shared
+ * skeleton tokens each hit freq 1.0.
+ *
+ * ## The histogram
+ *
+ * For every member page, all its landmark instances are tokenized and
+ * unioned into a single per-page token set (order-agnostic, deduped: a
+ * token appearing in two of the page's landmarks still counts once for
+ * that page). The corpus histogram is then "how many pages contain each
+ * token". Tokens that appear on nearly every page are the unit's chrome;
+ * tokens that appear on only a handful are page-specific content that
+ * happens to be tagged as a landmark.
+ *
+ * ## Why auto-cut instead of a hard-coded quorum
+ *
+ * A fixed 80% quorum (this file's earlier implementation) baked one
+ * threshold in for every unit. Real corpora don't obey a universal cutoff:
+ * a section-local landmark token that appears on 60% of a unit's pages is
+ * the section's chrome under any reasonable reading, but 80% quorum
+ * discards it. Auto-cut looks at the *shape* of the frequency distribution
+ * and picks the widest gap between adjacent frequencies — if the
+ * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
+ * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
+ * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
+ * "chrome for this unit". If instead the distribution is flat, the clamp
+ * to {@link SHELL_QUORUM_FALLBACK_FRACTION} keeps the threshold from being
+ * tighter than the fallback default.
+ *
+ * ## Fallbacks
+ *
+ * A single distinct token (`heights.length < 2`) or a perfectly flat
+ * distribution (`maxGap === 0`) returns the
+ * {@link SHELL_QUORUM_FALLBACK_FRACTION} clamp verbatim — exactly the same
+ * 80%-quorum behavior as before. So degenerate cases degrade to the old
+ * contract; only richer distributions get the auto-cut benefit.
+ *
+ * A page with no landmarks contributes an empty set, deliberately, so the
+ * shell-corroboration jaccard between two landmark-less pages is 0 rather
+ * than 1 (which it would be if we handed back a `<body></body>`-derived
+ * `{body}` fallback set to both sides).
+ *
+ * ## Reuse
+ *
+ * Originally private to {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * Stage B L2 corroboration; exported from its own module so
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}'s
+ * `includeLandmarkPositions` reporting path can run it once per final
+ * cluster to classify individual landmark instances as chrome (see
+ * {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}).
+ * @param perPageInstances
+ */
+export declare function shellQuorum(perPageInstances: readonly (readonly PerPageLandmarkInstance[])[]): ReadonlySet<string>;

package/dist/shell-quorum.js

@@ -0,0 +1,110 @@
+import { autoCutThreshold } from './auto-cut-threshold.js';
+/**
+ * Fallback clamp for {@link ./auto-cut-threshold.js | autoCutThreshold} when
+ * run on the per-landmark-instance token-frequency distribution in
+ * {@link ./shell-quorum.js | shellQuorum}. Independently tunable from
+ * `merge-cross-block-clusters.ts`'s own `QUORUM_FRACTION` (same value today,
+ * 0.8, by coincidence of both having been validated against the same real
+ * crawl corpora — not because the two are meant to move together).
+ */
+const SHELL_QUORUM_FALLBACK_FRACTION = 0.8;
+/**
+ * Discovers a unit's shell tokens by auto-cutting the per-*token* page-
+ * frequency histogram of every landmark instance's tokens. This is the same
+ * max-gap primitive used for Stage A merge-height cutoffs, applied
+ * recursively at the landmark-token layer.
+ *
+ * ## Why per-token and not per-signature
+ *
+ * An earlier iteration ran the histogram at the level of full landmark-
+ * instance signatures (canonicalized token sets). That failed on a real,
+ * common pattern: a shared site chrome whose markup carries a per-page
+ * distinguishing element (a breadcrumb, a page-title element with a page-
+ * specific class, a "current" state). All pages have most of the same
+ * tokens, but every page's full signature is distinct because tokens embed
+ * class names. Per-signature counting saw 5 signatures at freq 0.2 each,
+ * autoCutThreshold on the flat distribution returned the clamp, and the
+ * shell collapsed to empty even though every page shared the core header
+ * skeleton. Per-token counting handles the same case correctly — the shared
+ * skeleton tokens each hit freq 1.0.
+ *
+ * ## The histogram
+ *
+ * For every member page, all its landmark instances are tokenized and
+ * unioned into a single per-page token set (order-agnostic, deduped: a
+ * token appearing in two of the page's landmarks still counts once for
+ * that page). The corpus histogram is then "how many pages contain each
+ * token". Tokens that appear on nearly every page are the unit's chrome;
+ * tokens that appear on only a handful are page-specific content that
+ * happens to be tagged as a landmark.
+ *
+ * ## Why auto-cut instead of a hard-coded quorum
+ *
+ * A fixed 80% quorum (this file's earlier implementation) baked one
+ * threshold in for every unit. Real corpora don't obey a universal cutoff:
+ * a section-local landmark token that appears on 60% of a unit's pages is
+ * the section's chrome under any reasonable reading, but 80% quorum
+ * discards it. Auto-cut looks at the *shape* of the frequency distribution
+ * and picks the widest gap between adjacent frequencies — if the
+ * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65
+ * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around
+ * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as
+ * "chrome for this unit". If instead the distribution is flat, the clamp
+ * to {@link SHELL_QUORUM_FALLBACK_FRACTION} keeps the threshold from being
+ * tighter than the fallback default.
+ *
+ * ## Fallbacks
+ *
+ * A single distinct token (`heights.length < 2`) or a perfectly flat
+ * distribution (`maxGap === 0`) returns the
+ * {@link SHELL_QUORUM_FALLBACK_FRACTION} clamp verbatim — exactly the same
+ * 80%-quorum behavior as before. So degenerate cases degrade to the old
+ * contract; only richer distributions get the auto-cut benefit.
+ *
+ * A page with no landmarks contributes an empty set, deliberately, so the
+ * shell-corroboration jaccard between two landmark-less pages is 0 rather
+ * than 1 (which it would be if we handed back a `<body></body>`-derived
+ * `{body}` fallback set to both sides).
+ *
+ * ## Reuse
+ *
+ * Originally private to {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s
+ * Stage B L2 corroboration; exported from its own module so
+ * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}'s
+ * `includeLandmarkPositions` reporting path can run it once per final
+ * cluster to classify individual landmark instances as chrome (see
+ * {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}).
+ * @param perPageInstances
+ */
+export function shellQuorum(perPageInstances) {
+    const pageCount = perPageInstances.length;
+    if (pageCount === 0)
+        return new Set();
+    // Union all instance token sets per page (dedupe within page: a token
+    // present on both header and footer of the same page still counts once
+    // for that page's contribution).
+    const tokenPageCount = new Map();
+    for (const instances of perPageInstances) {
+        const perPageUnion = new Set();
+        for (const inst of instances) {
+            for (const token of inst.tokens)
+                perPageUnion.add(token);
+        }
+        for (const token of perPageUnion) {
+            tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1);
+        }
+    }
+    if (tokenPageCount.size === 0)
+        return new Set();
+    const frequencies = [];
+    for (const count of tokenPageCount.values()) {
+        frequencies.push(count / pageCount);
+    }
+    const cut = autoCutThreshold(frequencies, SHELL_QUORUM_FALLBACK_FRACTION);
+    const shell = new Set();
+    for (const [token, count] of tokenPageCount) {
+        if (count / pageCount >= cut)
+            shell.add(token);
+    }
+    return shell;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@d-zero/page-cluster",
-	"version": "0.3.0",
-	"description": "Tokenizes an HTML document's body into a structural signature for duplicate/near-duplicate page detection at crawl scale",
+	"version": "0.4.0",
+	"description": "Clusters crawled HTML pages by DOM-structure similarity — assigns the same key to pages sharing a template, ignoring text content. CLI-first, with library APIs.",
 	"author": "D-ZERO",
 	"license": "MIT",
 	"publishConfig": {
@@ -36,7 +36,7 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/dealer": "1.10.0",
+		"@d-zero/dealer": "1.10.1",
 		"@d-zero/shared": "0.22.2",
 		"htmlparser2": "12.0.0"
 	},
@@ -45,5 +45,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/page-cluster"
 	},
-	"gitHead": "5429d79837cca262ae5b74b32922eae6f7ef2649"
+	"gitHead": "2df2273542928b0bd3e4507c161f80a4fffca9e0"
 }