@cosmocoder/mcp-web-docs 2.1.6 → 2.1.7

package/build/crawler/login-page-signals.d.ts

@@ -0,0 +1,20 @@
+/** Read in the page, so it must not reference anything outside itself. */
+export declare function readLoginPageSignals(): {
+    text: string;
+    hasPasswordInput: boolean;
+    headings: string[];
+    frameSources: string[];
+};
+/** A login shell is nothing but a frame or a button; a docs page carrying a sign-in box has an article */
+export declare const LOGIN_SHELL_MAX_TEXT = 200;
+/** Three of the detector's six indicators. Stricter than its own two, which it never applies here. */
+export declare const LOGIN_PAGE_CONFIDENCE = 0.5;
+/**
+ * What tells this page apart from the next one, or null when nothing does. A theme rendering its
+ * headings as styled divs would otherwise make every page of the site one page.
+ *
+ * The page's own address comes out first, because a login page names where it is sending you back
+ * to. Only its own: removing anything else would fold pages differing by exactly that. Accepted
+ * cost: `POST /api/login` at /api/login and `POST /api/logout` at /api/logout fold into one.
+ */
+export declare function pageIdentity(headings: string[], text: string, currentUrl: string): string | null;

package/build/crawler/login-page-signals.js

@@ -0,0 +1,34 @@
+import { createHash } from 'node:crypto';
+/** Read in the page, so it must not reference anything outside itself. */
+export function readLoginPageSignals() {
+    const collapse = (value) => value.replace(/\s+/g, ' ').trim();
+    return {
+        text: collapse(document.body?.textContent || ''),
+        hasPasswordInput: document.querySelector('input[type="password"]') !== null,
+        headings: [...document.querySelectorAll('h1, h2')].slice(0, 5).map((heading) => collapse(heading.textContent || '')),
+        frameSources: [...document.querySelectorAll('iframe')].map((frame) => frame.getAttribute('src') || '').filter(Boolean),
+    };
+}
+/** A login shell is nothing but a frame or a button; a docs page carrying a sign-in box has an article */
+export const LOGIN_SHELL_MAX_TEXT = 200;
+/** Three of the detector's six indicators. Stricter than its own two, which it never applies here. */
+export const LOGIN_PAGE_CONFIDENCE = 0.5;
+/**
+ * What tells this page apart from the next one, or null when nothing does. A theme rendering its
+ * headings as styled divs would otherwise make every page of the site one page.
+ *
+ * The page's own address comes out first, because a login page names where it is sending you back
+ * to. Only its own: removing anything else would fold pages differing by exactly that. Accepted
+ * cost: `POST /api/login` at /api/login and `POST /api/logout` at /api/logout fold into one.
+ */
+export function pageIdentity(headings, text, currentUrl) {
+    const { pathname } = new URL(currentUrl);
+    const identifying = headings.map((heading) => heading.replaceAll(currentUrl, '').replaceAll(pathname, '').trim()).filter(Boolean);
+    if (identifying.length > 0) {
+        return createHash('sha1').update(JSON.stringify(identifying)).digest('hex');
+    }
+    // A page that is barely anything is still recognisable as that; one with an article and no
+    // headings could be any page.
+    return text.length < LOGIN_SHELL_MAX_TEXT ? 'shell' : null;
+}
+//# sourceMappingURL=login-page-signals.js.map

package/build/crawler/login-page-signals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"login-page-signals.js","sourceRoot":"","sources":["../../src/crawler/login-page-signals.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,0EAA0E;AAC1E,MAAM,UAAU,oBAAoB;IAMlC,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtE,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;QAChD,gBAAgB,EAAE,QAAQ,CAAC,aAAa,CAAC,wBAAwB,CAAC,KAAK,IAAI;QAC3E,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QACpH,YAAY,EAAE,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;KACvH,CAAC;AACJ,CAAC;AAED,0GAA0G;AAC1G,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAExC,sGAAsG;AACtG,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,QAAkB,EAAE,IAAY,EAAE,UAAkB;IAC/E,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClI,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IACD,2FAA2F;IAC3F,8BAA8B;IAC9B,OAAO,IAAI,CAAC,MAAM,GAAG,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC"}

package/build/crawler/login-page-signals.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/crawler/login-page-signals.test.js

@@ -0,0 +1,91 @@
+// @vitest-environment jsdom
+import { LOGIN_SHELL_MAX_TEXT, pageIdentity, readLoginPageSignals } from './login-page-signals.js';
+function render(html) {
+    document.body.innerHTML = html;
+    return readLoginPageSignals();
+}
+const LOGIN_FORM = `
+  <h1>Sign in</h1>
+  <form action="/login?next=%2Fdocs%2F42">
+    <input name="username" type="text">
+    <input name="password" type="password">
+  </form>`;
+describe('readLoginPageSignals', () => {
+    it('reads the text, the password input and the page shape', () => {
+        expect(render(LOGIN_FORM)).toEqual({
+            text: 'Sign in',
+            hasPasswordInput: true,
+            headings: ['Sign in'],
+            frameSources: [],
+        });
+    });
+    it('is the same for one login page answering different URLs', () => {
+        const first = render(`${LOGIN_FORM}<p>Continue to /docs/1. Attempt 1 of 5.</p>`);
+        const second = render(`${LOGIN_FORM.replace('%2Fdocs%2F42', '%2Fdocs%2F7')}<p>Continue to /docs/7. Attempt 2 of 5.</p>`);
+        expect(first.text).not.toEqual(second.text);
+        expect({ ...first, text: '' }).toEqual({ ...second, text: '' });
+    });
+    it('differs between documentation pages built from one template', () => {
+        const shapes = ['login', 'logout', 'refresh'].map((endpoint) => render(`<h1>POST /api/${endpoint}</h1><p>Sign in with your username and password.</p>`).headings);
+        expect(new Set(shapes.map((headings) => headings.join())).size).toBe(3);
+    });
+    it('reports an empty shape and no password input for an ordinary page', () => {
+        expect(render('<p>Ordinary content</p>')).toEqual({
+            text: 'Ordinary content',
+            hasPasswordInput: false,
+            headings: [],
+            frameSources: [],
+        });
+    });
+    it('reports the frames a page embeds, and only those with a source', () => {
+        expect(render('<iframe src="/sso/login"></iframe><iframe></iframe>').frameSources).toEqual(['/sso/login']);
+    });
+    // Rebuilding it from its own source is the only way a reference to anything outside it shows up
+    // here rather than in the browser page.evaluate sends it to
+    it('reads nothing from outside itself, as page.evaluate requires', () => {
+        document.body.innerHTML = LOGIN_FORM;
+        const detached = new Function(`return (${readLoginPageSignals.toString()})()`);
+        expect(detached()).toEqual(readLoginPageSignals());
+    });
+    it('collapses whitespace and caps the headings it reports', () => {
+        const signals = render(`<h1>  Sign\n  in  </h1>${[1, 2, 3, 4, 5, 6].map((n) => `<h2>Section ${n}</h2>`).join('')}`);
+        expect(signals.headings).toEqual(['Sign in', 'Section 1', 'Section 2', 'Section 3', 'Section 4']);
+    });
+});
+describe('pageIdentity', () => {
+    const url = 'https://example.com/docs/42';
+    const article = 'x'.repeat(LOGIN_SHELL_MAX_TEXT);
+    // A theme rendering its headings as styled divs leaves nothing to go on
+    it('has no answer for a page with an article on it and no headings', () => {
+        expect(pageIdentity([], article, url)).toBeNull();
+        expect(pageIdentity(['   '], article, url)).toBeNull();
+    });
+    // A page that is barely anything is still recognisable as that
+    it('recognises a bare page with no headings', () => {
+        expect(pageIdentity([], 'Continue with SSO', url)).toEqual(pageIdentity([], 'Continue', 'https://example.com/docs/7'));
+        expect(pageIdentity([], 'Continue', url)).not.toBeNull();
+    });
+    // A login shell is not always wordless - it explains why you are looking at it
+    it('still recognises a bare page that explains itself', () => {
+        expect(pageIdentity([], 'Your session has expired. Please sign in again to continue.', url)).not.toBeNull();
+        expect(pageIdentity([], 'x'.repeat(LOGIN_SHELL_MAX_TEXT - 1), url)).not.toBeNull();
+        expect(pageIdentity([], 'x'.repeat(LOGIN_SHELL_MAX_TEXT), url)).toBeNull();
+    });
+    // A login page names where it is sending you back to, so that part is not what the page is
+    it('is the same for one login page naming different URLs', () => {
+        expect(pageIdentity(['Sign in to continue to /docs/42'], article, url)).toEqual(pageIdentity(['Sign in to continue to /docs/7'], article, 'https://example.com/docs/7'));
+    });
+    it('differs between pages that are actually about different things', () => {
+        expect(pageIdentity(['Installing'], article, url)).not.toEqual(pageIdentity(['Configuring'], article, url));
+    });
+    // Only the page's own address comes out - a heading that IS an address elsewhere still counts
+    it('keeps an address that is not this page', () => {
+        expect(pageIdentity(['POST /api/login'], article, url)).not.toEqual(pageIdentity(['POST /api/logout'], article, url));
+    });
+    // Accepted: strip a heading that is only the page's own address and nothing is left. It takes a
+    // password field on each of these pages to matter, which the caller asks before this.
+    it('folds pages whose heading is only their own address', () => {
+        expect(pageIdentity(['POST /api/login'], article, 'https://example.com/api/login')).toEqual(pageIdentity(['POST /api/logout'], article, 'https://example.com/api/logout'));
+    });
+});
+//# sourceMappingURL=login-page-signals.test.js.map

package/build/crawler/login-page-signals.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"login-page-signals.test.js","sourceRoot":"","sources":["../../src/crawler/login-page-signals.test.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAC5B,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAEnG,SAAS,MAAM,CAAC,IAAY;IAC1B,QAAQ,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAC/B,OAAO,oBAAoB,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,GAAG;;;;;UAKT,CAAC;AAEX,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;YACjC,IAAI,EAAE,SAAS;YACf,gBAAgB,EAAE,IAAI;YACtB,QAAQ,EAAE,CAAC,SAAS,CAAC;YACrB,YAAY,EAAE,EAAE;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,UAAU,6CAA6C,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC,cAAc,EAAE,aAAa,CAAC,6CAA6C,CAAC,CAAC;QAEzH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,MAAM,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,GAAG,CAC/C,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,iBAAiB,QAAQ,sDAAsD,CAAC,CAAC,QAAQ,CAC/G,CAAC;QAEF,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC,CAAC,OAAO,CAAC;YAChD,IAAI,EAAE,kBAAkB;YACxB,gBAAgB,EAAE,KAAK;YACvB,QAAQ,EAAE,EAAE;YACZ,YAAY,EAAE,EAAE;SACjB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,MAAM,CAAC,qDAAqD,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;IAC7G,CAAC,CAAC,CAAC;IAEH,gGAAgG;IAChG,4DAA4D;IAC5D,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,QAAQ,CAAC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,WAAW,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAkD,CAAC;QAEhI,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAEpH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;IACpG,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,MAAM,GAAG,GAAG,6BAA6B,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAEjD,wEAAwE;IACxE,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAClD,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,+DAA+D;IAC/D,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,mBAAmB,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,4BAA4B,CAAC,CAAC,CAAC;QACvH,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,6DAA6D,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC5G,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,oBAAoB,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnF,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,2FAA2F;IAC3F,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CAAC,YAAY,CAAC,CAAC,iCAAiC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAC7E,YAAY,CAAC,CAAC,gCAAgC,CAAC,EAAE,OAAO,EAAE,4BAA4B,CAAC,CACxF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9G,CAAC,CAAC,CAAC;IAEH,8FAA8F;IAC9F,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,CAAC,YAAY,CAAC,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACxH,CAAC,CAAC,CAAC;IAEH,gGAAgG;IAChG,sFAAsF;IACtF,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,CAAC,YAAY,CAAC,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,+BAA+B,CAAC,CAAC,CAAC,OAAO,CACzF,YAAY,CAAC,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,gCAAgC,CAAC,CAC9E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmocoder/mcp-web-docs",
-  "version": "2.1.6",
+  "version": "2.1.7",
   "description": "MCP server for crawling and indexing web documentation - works with any website",
   "keywords": [
     "mcp",