npm - @d-zero/page-cluster - Versions diffs - 0.2.0 → 0.3.0 - Mend

@d-zero/page-cluster 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (51) hide show

package/LICENSE +21 -0
package/README.md +95 -41
package/dist/assign-contained-clusters.d.ts +42 -0
package/dist/assign-contained-clusters.js +156 -0
package/dist/auto-cut-threshold.d.ts +17 -0
package/dist/auto-cut-threshold.js +36 -0
package/dist/canonicalize-token-set.d.ts +17 -0
package/dist/canonicalize-token-set.js +19 -0
package/dist/cli.d.ts +39 -0
package/dist/cli.js +381 -0
package/dist/collapse-anonymous-divs.d.ts +21 -0
package/dist/collapse-anonymous-divs.js +42 -0
package/dist/complete-linkage-dendrogram.d.ts +41 -0
package/dist/complete-linkage-dendrogram.js +140 -0
package/dist/derive-comparison-sets.d.ts +22 -0
package/dist/derive-comparison-sets.js +33 -0
package/dist/derive-path-cluster-keys.d.ts +53 -0
package/dist/derive-path-cluster-keys.js +109 -0
package/dist/extract-landmarks.d.ts +91 -45
package/dist/extract-landmarks.js +122 -41
package/dist/filter-first-party-stylesheet-hrefs.d.ts +58 -24
package/dist/filter-first-party-stylesheet-hrefs.js +72 -33
package/dist/find-shallowest-elements.d.ts +48 -11
package/dist/find-shallowest-elements.js +41 -21
package/dist/merge-cross-block-clusters.d.ts +61 -0
package/dist/merge-cross-block-clusters.js +546 -0
package/dist/pass0-blocking.d.ts +89 -0
package/dist/pass0-blocking.js +87 -0
package/dist/per-page-landmark-signatures.d.ts +48 -0
package/dist/per-page-landmark-signatures.js +62 -0
package/dist/reservoir-sample.d.ts +43 -0
package/dist/reservoir-sample.js +98 -0
package/dist/resolve-blocking-group-keys.d.ts +8 -2
package/dist/resolve-blocking-group-keys.js +18 -4
package/dist/resolve-landmark-variant-keys.d.ts +41 -20
package/dist/resolve-landmark-variant-keys.js +69 -26
package/dist/resolve-page-cluster-keys.d.ts +292 -191
package/dist/resolve-page-cluster-keys.js +708 -157
package/dist/resolve-structural-cluster-keys.d.ts +9 -0
package/dist/resolve-structural-cluster-keys.js +14 -232
package/dist/shape-token.d.ts +11 -0
package/dist/shape-token.js +38 -0
package/dist/stage-a-per-block.d.ts +133 -0
package/dist/stage-a-per-block.js +178 -0
package/dist/tokenize.d.ts +6 -0
package/dist/tokenize.js +6 -0
package/package.json +5 -58
package/dist/html-region-utils.d.ts +0 -74
package/dist/html-region-utils.js +0 -96
package/dist/merge-landmark-affined-clusters.d.ts +0 -179
package/dist/merge-landmark-affined-clusters.js +0 -544

package/dist/cli.js ADDED Viewed

@@ -0,0 +1,381 @@
+#!/usr/bin/env node
+// Wire the `page-cluster` executable to the library's factory-based
+// `resolvePageClusterKeys`. Reads JSONL from stdin (one page per line),
+// writes JSONL to stdout (one cluster assignment per line, in input order),
+// and streams progress to stderr via `@d-zero/dealer`'s `Lanes` — in-place
+// animated header on a TTY, appended `[page-cluster] …` lines otherwise.
+import process from 'node:process';
+import { Lanes } from '@d-zero/dealer';
+import { resolvePageClusterKeys } from './resolve-page-cluster-keys.js';
+const HELP_TEXT = `Usage:
+  page-cluster [--content-block-attribute <name>] < pages.jsonl > clusters.jsonl
+Input (JSONL, one page per line):
+  {
+    "id": "any stable identifier",
+    "html": "<html>...</html>",
+    "paths": ["news", "1"],            // optional
+    "stylesheetHrefs": ["/a.css"],     // optional
+    "host": "example.com"              // optional
+  }
+Output (JSONL, one line per input page, in input order):
+  { "id": "...", "clusterKey": "..." }
+Options:
+  --content-block-attribute <name>   CMS-provided attribute marking freeform
+                                     content blocks that should be stripped
+                                     before comparison (e.g. \`data-bgb\`).
+  --help                             Print this help and exit.
+  --version                          Print the package version and exit.
+Progress:
+  Emitted to stderr while the run is in progress. On an interactive
+  terminal, an in-place animated header shows the current phase and
+  elapsed time. When stderr is not a TTY (redirected to a file, piped, or
+  under CI), each phase transition is appended as a \`[page-cluster] …\`
+  line so \`grep\` / \`awk\` stay trivial. Silence progress with
+  \`2>/dev/null\`; the JSONL output on stdout is unaffected either way.
+`;
+/**
+ * Log id used for the single-lane `lanes.update()` call under verbose mode.
+ * Lanes was originally designed for one line per parallel worker; here we
+ * only ever have one narrative line, so a fixed id is enough.
+ */
+const LANE_ID = 0;
+/**
+ * Prefix rendered ahead of every verbose progress line. Set on `lanes.header`
+ * at CLI init so that `lanes.update(LANE_ID, …)` under Lanes' verbose mode
+ * emits `[page-cluster] <line>` rather than the `undefined <line>` string
+ * it would produce with an unset header (see `dealer/lanes.ts:104`).
+ */
+const VERBOSE_HEADER = '[page-cluster]';
+/**
+ * Parses `process.argv`-style arguments (already sliced past `node script`)
+ * into a `CliArgs`. Deliberately tolerant of an unknown flag so the caller
+ * can decide the error message shape, and so tests can assert on the
+ * unrecognized flag name directly.
+ * @param argv
+ */
+export function parseArgs(argv) {
+    const out = {};
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        switch (arg) {
+            case '--help':
+            case '-h': {
+                out.help = true;
+                break;
+            }
+            case '--version':
+            case '-v': {
+                out.version = true;
+                break;
+            }
+            case '--content-block-attribute': {
+                const next = argv[i + 1];
+                if (next === undefined) {
+                    out.unknownFlag = `${arg} requires a value`;
+                    return out;
+                }
+                out.contentBlockAttribute = next;
+                i++;
+                break;
+            }
+            default: {
+                out.unknownFlag = arg;
+                return out;
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Streams `stdin` and yields per-line JSON-parsed page objects (plus the
+ * original line's `id`, preserved for the output row). Chunk-boundary
+ * splitting is done by hand rather than via `readline` because certain
+ * multi-KB UTF-8 lines have been observed to trip `readline`'s parser on
+ * this codebase — see `.page-cluster/scale-spike.mjs`'s note for the
+ * concrete case.
+ * @param input
+ */
+async function* readJsonlPages(input) {
+    input.setEncoding?.('utf8');
+    let leftover = '';
+    let lineNo = 0;
+    for await (const chunk of input) {
+        const text = leftover + chunk;
+        const parts = text.split('\n');
+        leftover = parts.pop() ?? '';
+        for (const line of parts) {
+            lineNo++;
+            if (line.length === 0)
+                continue;
+            let entry;
+            try {
+                entry = JSON.parse(line);
+            }
+            catch (error) {
+                throw new Error(`failed to parse JSONL line ${lineNo}: ${error.message}`);
+            }
+            if (typeof entry.html !== 'string') {
+                throw new TypeError(`JSONL line ${lineNo} is missing a string \`html\` field`);
+            }
+            yield {
+                id: entry.id,
+                page: {
+                    html: entry.html,
+                    paths: entry.paths ?? [],
+                    stylesheetHrefs: entry.stylesheetHrefs ?? [],
+                    host: entry.host,
+                },
+            };
+        }
+    }
+    if (leftover.length > 0) {
+        lineNo++;
+        let entry;
+        try {
+            entry = JSON.parse(leftover);
+        }
+        catch (error) {
+            throw new Error(`failed to parse final JSONL line ${lineNo}: ${error.message}`);
+        }
+        if (typeof entry.html !== 'string') {
+            throw new TypeError(`JSONL line ${lineNo} is missing a string \`html\` field`);
+        }
+        yield {
+            id: entry.id,
+            page: {
+                html: entry.html,
+                paths: entry.paths ?? [],
+                stylesheetHrefs: entry.stylesheetHrefs ?? [],
+                host: entry.host,
+            },
+        };
+    }
+}
+/**
+ * Renders the current lane message via TTY header or verbose-appended line
+ * depending on the `useTty` mode. Wraps both `lanes.header()` and
+ * `lanes.update(LANE_ID, …)` because — as documented in `dealer/lanes.ts` —
+ * `header()` under verbose is a state-only setter that produces no output,
+ * so verbose runs must go through `update()` to actually emit a line.
+ * @param lanes
+ * @param useTty
+ * @param line
+ */
+function renderProgress(lanes, useTty, line) {
+    if (useTty) {
+        lanes.header(line.tty);
+    }
+    else {
+        lanes.update(LANE_ID, line.verbose);
+    }
+}
+/**
+ * Human-facing wording for the "reading input pages" moment (before stdin
+ * is fully consumed). Verbose lines omit the `[page-cluster]` prefix — it
+ * is applied once via `VERBOSE_HEADER` when Lanes constructs each verbose
+ * line.
+ */
+const READING_INPUT = {
+    tty: '%earth% page-cluster — reading input...',
+    verbose: 'reading input pages...',
+};
+/**
+ * Progress line emitted after stdin is fully consumed, before the async
+ * factory-based `resolvePageClusterKeys` starts producing events.
+ * @param pageCount
+ */
+function readingDoneLine(pageCount) {
+    return {
+        tty: `%earth% page-cluster — read ${pageCount} pages, clustering...`,
+        verbose: `read ${pageCount} pages, clustering...`,
+    };
+}
+/**
+ * Final summary line rendered right before `lanes.close()`.
+ * @param pageCount
+ * @param clusterCount
+ * @param elapsedSec
+ */
+function doneLine(pageCount, clusterCount, elapsedSec) {
+    const body = `${pageCount} pages in ${clusterCount} clusters (elapsed ${elapsedSec}s)`;
+    return {
+        tty: `page-cluster — done: ${body}`,
+        verbose: `done — ${body}`,
+    };
+}
+/**
+ * Fatal-error line rendered via Lanes rather than a direct
+ * `stderr.write` — the interactive Display's `close()` runs a
+ * CURSOR_UP + ERASE_DOWN repaint that would wipe any raw stderr write
+ * emitted before it, silently swallowing the diagnostic in TTY mode.
+ * Sending the error through Lanes puts it into the final repainted
+ * frame, so it survives `close()`.
+ * @param message
+ */
+function errorLine(message) {
+    return {
+        tty: `page-cluster: ${message}`,
+        verbose: `error: ${message}`,
+    };
+}
+/**
+ * Maps a library `ProgressEvent` to a human-facing `ProgressLine`. The
+ * verbose arm keeps the historical `pass0:` / `pass1:` / `pass1b:` /
+ * `stage-b:` phase tokens so callers who grep stderr by phase name (a
+ * pattern the earlier `formatProgress` documented) stay compatible; TTY
+ * lines use natural-language wording since interactive users read the
+ * header, not grep.
+ * @param event
+ * @param elapsedSec
+ */
+function formatProgressLine(event, elapsedSec) {
+    switch (event.phase) {
+        case 'pass0-signals': {
+            return {
+                tty: `%earth% page-cluster — reading signals: ${event.pagesSeen} pages (elapsed ${elapsedSec}s)`,
+                verbose: `pass0: ${event.pagesSeen} pages read`,
+            };
+        }
+        case 'pass1-block-complete': {
+            return {
+                tty: `%earth% page-cluster — clustering ${event.blocksProcessed}/${event.totalBlocks} blocks (elapsed ${elapsedSec}s)`,
+                verbose: `pass1: clustered block ${event.blocksProcessed}/${event.totalBlocks}`,
+            };
+        }
+        case 'pass1b-assign': {
+            return {
+                tty: `%earth% page-cluster — assigning pages: ${event.pagesAssigned}/${event.pagesToAssign} (elapsed ${elapsedSec}s)`,
+                verbose: `pass1b: ${event.pagesAssigned}/${event.pagesToAssign} pages assigned`,
+            };
+        }
+        case 'stage-b-start': {
+            return {
+                tty: `%earth% page-cluster — merging ${event.unitCount} units (elapsed ${elapsedSec}s)`,
+                verbose: `stage-b: merging ${event.unitCount} units`,
+            };
+        }
+    }
+}
+/**
+ * Test-friendly entry point: takes the run's stdin/stdout/stderr streams
+ * and the parsed CLI flags rather than reading them out of the process
+ * globals. `runCli` returns the exit code, allowing the caller (either the
+ * top-level `main` here or a spec test) to decide how to signal it.
+ * @param options
+ * @param options.stdin
+ * @param options.stdout
+ * @param options.stderr
+ * @param options.argv
+ * @param options.version
+ */
+export async function runCli(options) {
+    const args = parseArgs(options.argv);
+    if (args.help) {
+        options.stdout.write(HELP_TEXT);
+        return 0;
+    }
+    if (args.version) {
+        options.stdout.write(`${options.version}\n`);
+        return 0;
+    }
+    if (args.unknownFlag !== undefined) {
+        options.stderr.write(`page-cluster: unrecognized argument ${JSON.stringify(args.unknownFlag)}\n`);
+        options.stderr.write(HELP_TEXT);
+        return 2;
+    }
+    // TTY detection: `NodeJS.WritableStream` doesn't expose `isTTY`, but the
+    // concrete `WriteStream` (and the in-memory `Writable` collectors tests
+    // pass in) does — reading it defensively lets both real usage and
+    // unit-test doubles work without a separate `--no-progress` flag.
+    const useTty = options.stderr.isTTY === true;
+    const lanes = new Lanes({ stream: options.stderr, verbose: !useTty });
+    // Verbose Lanes prepends `#header` to every `update()` line. Without
+    // this seed call the header would be undefined and each progress line
+    // would begin with the literal string `undefined ` — bug caught by
+    // code-review high effort. TTY mode overwrites the header per event
+    // so the seed value is only surfaced verbatim under verbose.
+    if (!useTty) {
+        lanes.header(VERBOSE_HEADER);
+    }
+    const startTime = Date.now();
+    const elapsed = () => Math.max(0, Math.round((Date.now() - startTime) / 1000));
+    // Every early return past this point must run through the finally block
+    // so `lanes.close()` releases the display's setTimeout timer — without
+    // it a `return 1` on a stdin parse error would leave the process
+    // hanging on the timer's next tick.
+    try {
+        renderProgress(lanes, useTty, READING_INPUT);
+        // Load every JSONL line into memory once so the ids array stays
+        // parallel to the pages array — the streaming driver reads its
+        // factory twice, and stdin is a one-shot pipe.
+        const ids = [];
+        const pages = [];
+        try {
+            for await (const { id, page } of readJsonlPages(options.stdin)) {
+                ids.push(id);
+                pages.push(page);
+            }
+        }
+        catch (error) {
+            renderProgress(lanes, useTty, errorLine(error.message));
+            return 1;
+        }
+        renderProgress(lanes, useTty, readingDoneLine(pages.length));
+        const resolveOptions = {
+            contentBlockAttribute: args.contentBlockAttribute,
+            onProgress: (event) => {
+                renderProgress(lanes, useTty, formatProgressLine(event, elapsed()));
+            },
+        };
+        let keys;
+        try {
+            keys = await resolvePageClusterKeys(() => pages, resolveOptions);
+        }
+        catch (error) {
+            renderProgress(lanes, useTty, errorLine(error.message));
+            return 1;
+        }
+        const clusterCount = new Set(keys).size;
+        renderProgress(lanes, useTty, doneLine(pages.length, clusterCount, elapsed()));
+        for (const [index, key] of keys.entries()) {
+            options.stdout.write(`${JSON.stringify({ id: ids[index] ?? index, clusterKey: key })}\n`);
+        }
+        return 0;
+    }
+    finally {
+        lanes.close();
+    }
+}
+/**
+ * Reads the package version out of `package.json` at runtime. Kept as a
+ * separate helper so `runCli` can be exercised in tests without needing a
+ * real `package.json` on disk.
+ */
+async function readPackageVersion() {
+    try {
+        const url = new URL('../package.json', import.meta.url);
+        const { readFile } = await import('node:fs/promises');
+        const raw = await readFile(url, 'utf8');
+        const parsed = JSON.parse(raw);
+        return parsed.version ?? '0.0.0';
+    }
+    catch {
+        return '0.0.0';
+    }
+}
+// Only run when invoked as the actual entry point.
+if (import.meta.url === `file://${process.argv[1]}`) {
+    const version = await readPackageVersion();
+    const exitCode = await runCli({
+        stdin: process.stdin,
+        stdout: process.stdout,
+        stderr: process.stderr,
+        argv: process.argv.slice(2),
+        version,
+    });
+    process.exit(exitCode);
+}

package/dist/collapse-anonymous-divs.d.ts ADDED Viewed

@@ -0,0 +1,21 @@
+/**
+ * Removes bare foldable-tag segments (e.g. plain `div`, `span`) from the
+ * interior of a path token, leaving the first and last segments untouched.
+ * Used before containment-assignment union construction so that two clusters
+ * whose paths differ only in intermediate anonymous wrapper `<div>`s are not
+ * spuriously separated.
+ *
+ * Intermediate bare `div`/`span` elements are anonymous wrappers with no
+ * class, role, or type — they carry no structural meaning beyond "something
+ * was nested here", and their presence varies freely across CMS themes and
+ * minor template revisions. Stripping them from the middle of a path keeps
+ * the union comparison focused on the meaningful skeleton (semantic tags,
+ * class-bearing wrappers, bracketed roles).
+ *
+ * First and last segments are always preserved: the first segment anchors the
+ * path in the document tree (`body`, `main`, …) and the last segment is the
+ * leaf element being tokenized — both carry meaning even when they are bare
+ * foldable tags.
+ * @param token A full path token, e.g. `body>main>div>section.c-x>div>.card`.
+ */
+export declare function collapseAnonymousDivs(token: string): string;

package/dist/collapse-anonymous-divs.js ADDED Viewed

@@ -0,0 +1,42 @@
+import { FOLDABLE_TAGS } from './foldable-tags.js';
+/**
+ * Only `FOLDABLE_TAGS` qualify — those are the tags `build-segment.ts` omits
+ * the name from when they have classes (producing `.c-x` instead of `div.c-x`),
+ * so a bare `div` or `span` with NO class is a structural-only wrapper.
+ * Non-foldable tags (`section`, `article`, …) always appear with their tag name
+ * and carry semantic meaning even when classless, so they are kept.
+ * @param segment
+ */
+function isAnonymousSegment(segment) {
+    return FOLDABLE_TAGS.has(segment);
+}
+/**
+ * Removes bare foldable-tag segments (e.g. plain `div`, `span`) from the
+ * interior of a path token, leaving the first and last segments untouched.
+ * Used before containment-assignment union construction so that two clusters
+ * whose paths differ only in intermediate anonymous wrapper `<div>`s are not
+ * spuriously separated.
+ *
+ * Intermediate bare `div`/`span` elements are anonymous wrappers with no
+ * class, role, or type — they carry no structural meaning beyond "something
+ * was nested here", and their presence varies freely across CMS themes and
+ * minor template revisions. Stripping them from the middle of a path keeps
+ * the union comparison focused on the meaningful skeleton (semantic tags,
+ * class-bearing wrappers, bracketed roles).
+ *
+ * First and last segments are always preserved: the first segment anchors the
+ * path in the document tree (`body`, `main`, …) and the last segment is the
+ * leaf element being tokenized — both carry meaning even when they are bare
+ * foldable tags.
+ * @param token A full path token, e.g. `body>main>div>section.c-x>div>.card`.
+ */
+export function collapseAnonymousDivs(token) {
+    const segments = token.split('>');
+    if (segments.length <= 2) {
+        return token;
+    }
+    const first = segments[0] ?? '';
+    const last = segments.at(-1) ?? '';
+    const middle = segments.slice(1, -1).filter((seg) => !isAnonymousSegment(seg));
+    return [first, ...middle, last].join('>');
+}

package/dist/complete-linkage-dendrogram.d.ts ADDED Viewed

@@ -0,0 +1,41 @@
+/**
+ * One merge event in the complete-linkage dendrogram. `survivor` is the
+ * lower index (kept active), `dead` the higher (deactivated), `height`
+ * is the Jaccard similarity at which the merge occurred. All `n - 1`
+ * merges for `n` input sets are recorded, including those below any
+ * particular threshold — the caller decides which heights are meaningful
+ * via `labelsAtThreshold`.
+ */
+export type DendrogramMerge = {
+    readonly survivor: number;
+    readonly dead: number;
+    readonly height: number;
+};
+/**
+ * Computes the complete-linkage dendrogram for `tokenSets` via the NN-chain
+ * algorithm (Murtagh, F., 1983) and returns all `n - 1` merge events in
+ * discovery order. This is the same O(n²) NN-chain implementation as
+ * `resolve-structural-cluster-keys.ts` — extracted here so that callers can
+ * record the full height sequence and apply an arbitrary threshold cut via
+ * `labelsAtThreshold`, rather than committing to a fixed threshold upfront.
+ *
+ * Recording all merges (not just threshold-clearing ones) is essential for
+ * `autoCutThreshold`: to find the largest gap in the height sequence, every
+ * height must be available regardless of where the eventual cut lands.
+ * @param tokenSets
+ */
+export declare function completeLinkageDendrogram(tokenSets: readonly ReadonlySet<string>[]): DendrogramMerge[];
+/**
+ * Reconstructs cluster labels for all `size` original items at a given
+ * similarity threshold, using the merge list from `completeLinkageDendrogram`.
+ *
+ * Lance-Williams monotonicity guarantees that only merges at `>= threshold`
+ * need to be applied to reconstruct the correct threshold cut, regardless of
+ * the order in which NN-chain discovered them. This is the same invariant
+ * documented in `resolve-structural-cluster-keys.ts`'s JSDoc for
+ * `clusterByCompleteLinkage`.
+ * @param size Total number of items (= length of the original `tokenSets`).
+ * @param merges All merge events from `completeLinkageDendrogram`.
+ * @param threshold Similarity threshold; merges below this are ignored.
+ */
+export declare function labelsAtThreshold(size: number, merges: readonly DendrogramMerge[], threshold: number): number[];

package/dist/complete-linkage-dendrogram.js ADDED Viewed

@@ -0,0 +1,140 @@
+import { jaccardSimilarity } from './jaccard-similarity.js';
+/**
+ * `jaccardSimilarity()` returns a floating-point division that can land a
+ * hair below the caller's threshold even when mathematically equal. Same
+ * epsilon as `resolve-structural-cluster-keys.ts`.
+ */
+const BOUNDARY_EPSILON = 1e-9;
+/**
+ * `noUncheckedIndexedAccess` makes indexed reads return `T | undefined`.
+ * Call sites here index positions this function itself generated (NN-chain
+ * internal arrays), so the throw branch is unreachable in practice — it exists
+ * solely to satisfy the compiler without a non-null assertion everywhere.
+ * @param values
+ * @param index
+ */
+function requireIndex(values, index) {
+    const value = values[index];
+    if (value === undefined) {
+        throw new Error('completeLinkageDendrogram: index out of bounds');
+    }
+    return value;
+}
+/**
+ * Path-compressed union-find root lookup. `labelsAtThreshold` rebuilds the
+ * cluster label for each item by following its parent chain to the root;
+ * path compression keeps repeated lookups O(α) instead of O(n).
+ * @param parent
+ * @param index
+ */
+function find(parent, index) {
+    let root = index;
+    while (requireIndex(parent, root) !== root) {
+        root = requireIndex(parent, root);
+    }
+    let current = index;
+    while (current !== root) {
+        const next = requireIndex(parent, current);
+        parent[current] = root;
+        current = next;
+    }
+    return root;
+}
+/**
+ * Computes the complete-linkage dendrogram for `tokenSets` via the NN-chain
+ * algorithm (Murtagh, F., 1983) and returns all `n - 1` merge events in
+ * discovery order. This is the same O(n²) NN-chain implementation as
+ * `resolve-structural-cluster-keys.ts` — extracted here so that callers can
+ * record the full height sequence and apply an arbitrary threshold cut via
+ * `labelsAtThreshold`, rather than committing to a fixed threshold upfront.
+ *
+ * Recording all merges (not just threshold-clearing ones) is essential for
+ * `autoCutThreshold`: to find the largest gap in the height sequence, every
+ * height must be available regardless of where the eventual cut lands.
+ * @param tokenSets
+ */
+export function completeLinkageDendrogram(tokenSets) {
+    const size = tokenSets.length;
+    if (size <= 1)
+        return [];
+    const similarity = new Float64Array(size * size);
+    for (let i = 0; i < size; i++) {
+        for (let j = i + 1; j < size; j++) {
+            const score = jaccardSimilarity(requireIndex(tokenSets, i), requireIndex(tokenSets, j));
+            similarity[i * size + j] = score;
+            similarity[j * size + i] = score;
+        }
+    }
+    const active = new Uint8Array(size).fill(1);
+    const chain = [];
+    const merges = [];
+    const findFreshStart = () => {
+        for (let index = 0; index < size; index++) {
+            if (requireIndex(active, index) === 1)
+                return index;
+        }
+        throw new Error('completeLinkageDendrogram: no active cluster left to resume from');
+    };
+    let activeCount = size;
+    while (activeCount > 1) {
+        if (chain.length === 0)
+            chain.push(findFreshStart());
+        const top = requireIndex(chain, chain.length - 1);
+        let best = -1;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (let candidate = 0; candidate < size; candidate++) {
+            if (candidate !== top && requireIndex(active, candidate) === 1) {
+                const score = requireIndex(similarity, top * size + candidate);
+                if (score > bestScore) {
+                    bestScore = score;
+                    best = candidate;
+                }
+            }
+        }
+        const secondFromTop = chain.length >= 2 ? chain.at(-2) : undefined;
+        if (best === secondFromTop) {
+            chain.pop();
+            chain.pop();
+            const survivor = Math.min(top, best);
+            const dead = Math.max(top, best);
+            for (let candidate = 0; candidate < size; candidate++) {
+                if (candidate !== top &&
+                    candidate !== best &&
+                    requireIndex(active, candidate) === 1) {
+                    const merged = Math.min(requireIndex(similarity, top * size + candidate), requireIndex(similarity, best * size + candidate));
+                    similarity[survivor * size + candidate] = merged;
+                    similarity[candidate * size + survivor] = merged;
+                }
+            }
+            active[dead] = 0;
+            merges.push({ survivor, dead, height: bestScore });
+            activeCount--;
+        }
+        else {
+            chain.push(best);
+        }
+    }
+    return merges;
+}
+/**
+ * Reconstructs cluster labels for all `size` original items at a given
+ * similarity threshold, using the merge list from `completeLinkageDendrogram`.
+ *
+ * Lance-Williams monotonicity guarantees that only merges at `>= threshold`
+ * need to be applied to reconstruct the correct threshold cut, regardless of
+ * the order in which NN-chain discovered them. This is the same invariant
+ * documented in `resolve-structural-cluster-keys.ts`'s JSDoc for
+ * `clusterByCompleteLinkage`.
+ * @param size Total number of items (= length of the original `tokenSets`).
+ * @param merges All merge events from `completeLinkageDendrogram`.
+ * @param threshold Similarity threshold; merges below this are ignored.
+ */
+export function labelsAtThreshold(size, merges, threshold) {
+    const parent = Int32Array.from({ length: size }, (_, i) => i);
+    for (const { survivor, dead, height } of merges) {
+        if (height >= threshold - BOUNDARY_EPSILON) {
+            parent[find(parent, dead)] = find(parent, survivor);
+        }
+    }
+    return Array.from({ length: size }, (_, i) => find(parent, i));
+}

package/dist/derive-comparison-sets.d.ts ADDED Viewed

@@ -0,0 +1,22 @@
+/**
+ * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency`
+ * (the default 90% cutoff) degenerate rather than usefully separate chrome
+ * from content — see `deriveComparisonSets` for the failure mode. Derived
+ * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly
+ * one page out of `n` still counts as chrome only if
+ * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, the caller should fall
+ * back to comparing token sets directly, unfiltered.
+ */
+export declare const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10;
+/**
+ * Narrows each page's token set to its page-specific content before
+ * clustering, so pages that only share site-wide chrome (header/nav/footer)
+ * don't read as more similar than they structurally are. Skipped below
+ * `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages — see that constant's JSDoc.
+ *
+ * A page whose entire token set narrows away falls back to its raw tokens
+ * rather than the empty result, for the same reason documented in
+ * `resolve-structural-cluster-keys.ts`.
+ * @param tokenSets
+ */
+export declare function deriveComparisonSets(tokenSets: readonly ReadonlySet<string>[]): readonly ReadonlySet<string>[];

package/dist/derive-comparison-sets.js ADDED Viewed

@@ -0,0 +1,33 @@
+import { computeDocumentFrequency } from './compute-document-frequency.js';
+import { splitTokensByFrequency } from './split-tokens-by-frequency.js';
+/**
+ * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency`
+ * (the default 90% cutoff) degenerate rather than usefully separate chrome
+ * from content — see `deriveComparisonSets` for the failure mode. Derived
+ * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly
+ * one page out of `n` still counts as chrome only if
+ * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, the caller should fall
+ * back to comparing token sets directly, unfiltered.
+ */
+export const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10;
+/**
+ * Narrows each page's token set to its page-specific content before
+ * clustering, so pages that only share site-wide chrome (header/nav/footer)
+ * don't read as more similar than they structurally are. Skipped below
+ * `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages — see that constant's JSDoc.
+ *
+ * A page whose entire token set narrows away falls back to its raw tokens
+ * rather than the empty result, for the same reason documented in
+ * `resolve-structural-cluster-keys.ts`.
+ * @param tokenSets
+ */
+export function deriveComparisonSets(tokenSets) {
+    if (tokenSets.length < MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) {
+        return tokenSets;
+    }
+    const corpusFrequency = computeDocumentFrequency(tokenSets);
+    return tokenSets.map((tokens) => {
+        const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency);
+        return contentTokens.size === 0 && tokens.size > 0 ? tokens : contentTokens;
+    });
+}