@slicedearth/whoisleuth-cli 2.1.0 → 2.3.0

package/lib/security-txt.mjs

@@ -50,6 +50,38 @@ function normalizedPublishedUri(value, schemes) {
             return null;
         if (url.protocol === 'https:' && !url.hostname)
             return null;
+        if (url.protocol === 'mailto:') {
+            let recipients;
+            try {
+                recipients = decodeURIComponent(url.pathname).split(',');
+            }
+            catch {
+                return null;
+            }
+            if (!recipients.length || recipients.some((recipient) => {
+                const separator = recipient.lastIndexOf('@');
+                if (separator <= 0 || separator === recipient.length - 1)
+                    return true;
+                const local = recipient.slice(0, separator);
+                const mailDomain = recipient.slice(separator + 1).toLowerCase();
+                return !/^[a-z0-9.!#$%&'*+/=?^_`{|}~-]+$/iu.test(local)
+                    || local.startsWith('.') || local.endsWith('.') || local.includes('..')
+                    || !canonicalHostname(mailDomain);
+            }))
+                return null;
+        }
+        if (url.protocol === 'tel:') {
+            let number;
+            try {
+                number = decodeURIComponent(url.pathname);
+            }
+            catch {
+                return null;
+            }
+            if (!/^\+?[0-9(). -]+(?:;[a-z0-9-]+=[a-z0-9.-]+)*$/iu.test(number)
+                || (number.match(/[0-9]/gu)?.length ?? 0) < 3)
+                return null;
+        }
         url.search = '';
         url.hash = '';
         return url.toString().slice(0, MAX_SECURITY_TXT_URL_LENGTH);
@@ -211,7 +243,7 @@ function parseSecurityTxt(text, options = {}) {
     const finalComparisonUrl = options.finalUrl ? normalizedCanonicalComparisonUrl(options.finalUrl) : null;
     const canonicalMatches = boundedCanonical.values.length && finalUrl && finalComparisonUrl
         ? canonicalComparison.slice(0, MAX_SECURITY_TXT_VALUES)
-            .some((value) => value.toLowerCase() === finalComparisonUrl.toLowerCase())
+            .some((value) => value === finalComparisonUrl)
         : null;
     const stale = Boolean(expiresAt && Date.parse(expiresAt) <= (options.now ?? Date.now()));
     const requiredMalformed = boundedContacts.values.length === 0 || expires.length !== 1;

package/lib/static-html-analysis.mjs

@@ -73,7 +73,7 @@ const JAVASCRIPT_MEDIA_TYPES = new Set([
     'text/javascript1.1', 'text/javascript1.2', 'text/javascript1.3', 'text/javascript1.4',
     'text/javascript1.5', 'text/jscript', 'text/livescript', 'text/x-ecmascript', 'text/x-javascript',
 ]);
-function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
+function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH, lowercase = true) {
     const attribute = attributes
         .slice(0, MAX_ATTRIBUTES_PER_TAG)
         .find((candidate) => candidate.name.toLowerCase() === name);
@@ -84,7 +84,7 @@ function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LEN
     }
     return {
         present: true,
-        value: attribute.value.trim().toLowerCase(),
+        value: lowercase ? attribute.value.trim().toLowerCase() : attribute.value.trim(),
         truncated: attributes.length > MAX_ATTRIBUTES_PER_TAG,
     };
 }
@@ -116,8 +116,8 @@ function formMethod(attributes) {
         truncated: method.truncated,
     };
 }
-function formAction(attributes, baseUrl) {
-    const action = attributeValue(attributes, 'action');
+function formAction(attributes, resolutionBase, documentOrigin) {
+    const action = attributeValue(attributes, 'action', MAX_FORM_ATTRIBUTE_LENGTH, false);
     if (!action.present && action.truncated) {
         return { relationship: 'unclassified', cleartext: false, truncated: true };
     }
@@ -127,16 +127,16 @@ function formAction(attributes, baseUrl) {
     if (action.value === null) {
         return { relationship: 'unclassified', cleartext: false, truncated: true };
     }
-    if (!baseUrl) {
+    if (!resolutionBase || !documentOrigin) {
         return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
     }
     try {
-        const parsed = new URL(action.value, baseUrl);
+        const parsed = new URL(action.value, resolutionBase);
         if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
             return { relationship: 'unclassified', cleartext: false, truncated: action.truncated };
         }
         return {
-            relationship: parsed.origin === baseUrl.origin ? 'sameOrigin' : 'external',
+            relationship: parsed.origin === documentOrigin ? 'sameOrigin' : 'external',
             cleartext: parsed.protocol === 'http:',
             truncated: action.truncated,
         };
@@ -238,7 +238,9 @@ function analyzeStaticHtml(value, options = {}) {
     const supplied = typeof value === 'string' ? value : '';
     const inputLimitReached = supplied.length > MAX_STATIC_HTML_CHARS;
     const html = supplied.slice(0, MAX_STATIC_HTML_CHARS);
-    const baseUrl = safeBaseUrl(options.baseUrl);
+    const documentUrl = safeBaseUrl(options.baseUrl);
+    let effectiveBaseUrl = documentUrl;
+    let baseHrefState = 'absent';
     const includeVisibleText = options.includeVisibleText === true;
     const markup = [];
     const visibleTextParts = [];
@@ -316,6 +318,17 @@ function analyzeStaticHtml(value, options = {}) {
     let scriptElementSeen = false;
     let nonVisibleTextDepth = 0;
     let visibleTextCharacters = 0;
+    let titleDepth = 0;
+    let titleCharacters = 0;
+    const titleParts = [];
+    const pendingFormActions = [];
+    function appendTitle(chars) {
+        if (titleDepth === 0 || !chars || titleCharacters >= 201)
+            return;
+        const retained = chars.slice(0, 201 - titleCharacters);
+        titleParts.push(retained);
+        titleCharacters += retained.length;
+    }
     function appendInlineScript(chars) {
         if (!activeInlineScript || !chars)
             return;
@@ -374,6 +387,29 @@ function analyzeStaticHtml(value, options = {}) {
                 implicitHeadTextDepth += 1;
             }
             const inPublicationHead = insideExplicitHead || (!headScopeClosed && !bodyContentStarted && HEAD_CONTEXT_TAGS.has(tagName));
+            if (tagName === 'base' && inPublicationHead && baseHrefState === 'absent') {
+                const href = attributeValue(token.attrs, 'href', MAX_FORM_ATTRIBUTE_LENGTH, false);
+                if (href.truncated && !href.present) {
+                    effectiveBaseUrl = documentUrl;
+                    baseHrefState = 'invalid';
+                }
+                else if (href.present) {
+                    try {
+                        const parsed = href.value !== null && documentUrl ? new URL(href.value, documentUrl) : null;
+                        if (!parsed || !['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
+                            throw new Error('Invalid document base URL.');
+                        }
+                        effectiveBaseUrl = parsed;
+                        baseHrefState = 'valid';
+                    }
+                    catch {
+                        effectiveBaseUrl = documentUrl;
+                        baseHrefState = 'invalid';
+                    }
+                }
+            }
+            if (tagName === 'title' && titleDepth === 0 && titleParts.length === 0)
+                titleDepth = 1;
             if (/^h[1-6]$/u.test(tagName)) {
                 const key = tagName;
                 publicationMetadata.headings[key] += 1;
@@ -566,11 +602,8 @@ function analyzeStaticHtml(value, options = {}) {
                     forms.formsObserved += 1;
                     const method = formMethod(token.attrs);
                     forms.methods[method.value] += 1;
-                    const action = formAction(token.attrs, baseUrl);
-                    forms.actions[action.relationship] += 1;
-                    if (action.cleartext)
-                        forms.actions.cleartext += 1;
-                    if (method.truncated || action.truncated)
+                    pendingFormActions.push(token.attrs);
+                    if (method.truncated || token.attrs.length > MAX_ATTRIBUTES_PER_TAG)
                         forms.truncated = true;
                 }
             }
@@ -612,6 +645,8 @@ function analyzeStaticHtml(value, options = {}) {
             const tagName = token.tagName.toLowerCase();
             if (tagName === 'script')
                 activeInlineScript = null;
+            if (tagName === 'title' && titleDepth > 0)
+                titleDepth = 0;
             if (tagName === 'head') {
                 insideExplicitHead = false;
                 headScopeClosed = true;
@@ -630,6 +665,7 @@ function analyzeStaticHtml(value, options = {}) {
                 headScopeClosed = true;
             }
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onNullCharacter(token) {
@@ -638,10 +674,12 @@ function analyzeStaticHtml(value, options = {}) {
                 headScopeClosed = true;
             }
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onWhitespaceCharacter(token) {
             appendInlineScript(token.chars);
+            appendTitle(token.chars);
             appendVisibleText(token.chars);
         },
         onComment() { },
@@ -650,6 +688,16 @@ function analyzeStaticHtml(value, options = {}) {
     };
     tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
     tokenizer.write(html, true);
+    for (const attributes of pendingFormActions) {
+        const action = formAction(attributes, effectiveBaseUrl, documentUrl?.origin ?? null);
+        forms.actions[action.relationship] += 1;
+        if (action.cleartext)
+            forms.actions.cleartext += 1;
+        if (action.truncated)
+            forms.truncated = true;
+    }
+    if (baseHrefState === 'invalid')
+        forms.truncated = true;
     publicationMetadata.truncated = publicationMetadata.truncated || inputLimitReached || publicationTagLimitReached;
     publicationMetadata.documentTruncated = inputLimitReached || publicationTagLimitReached;
     if (publicationMetadata.documentTruncated) {
@@ -671,6 +719,14 @@ function analyzeStaticHtml(value, options = {}) {
     publicationMetadata.renderBlockingCandidates.total = publicationMetadata.renderBlockingCandidates.script
         + publicationMetadata.renderBlockingCandidates.stylesheet;
     return {
+        title: (() => {
+            const value = titleParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\ufffd/gu, ' ').replace(/\s+/gu, ' ').trim();
+            if (!value)
+                return null;
+            return value.length > 200 ? `${value.slice(0, 200)}…` : value;
+        })(),
+        effectiveBaseUrl: effectiveBaseUrl?.toString() ?? null,
+        baseHrefState,
         markup: markup.join('\n'),
         visibleText: visibleTextParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\s+/gu, ' '),
         structureTokens,

package/lib/website-technology.mjs

@@ -9,6 +9,23 @@ import { createObservation } from "../packages/evidence/observation.mjs";
 import { MAX_STATIC_HTML_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, } from "./static-html-analysis.mjs";
 import { TECHNOLOGY_EVIDENCE_ROLE_ORDER, } from "./technology-evidence-role.mjs";
 import { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, TECHNOLOGY_PROFILE_VERSION, } from "./lookup-child-profile-contract.mjs";
+const TECHNOLOGY_CATEGORIES = Object.freeze([
+    'application runtime',
+    'content management',
+    'commerce',
+    'site builder',
+    'web framework',
+    'static site generator',
+    'web server',
+    'delivery platform',
+]);
+const TECHNOLOGY_EVIDENCE_SOURCES = Object.freeze([
+    'generator metadata',
+    'static HTML',
+    'resource origin',
+    'HTTP server header',
+    'passive response header',
+]);
 const MAX_TECHNOLOGY_HTML_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_TECHNOLOGY_TAG_LENGTH = MAX_TAG_LENGTH;
 const MAX_RESOURCE_ORIGINS = 30;
@@ -38,6 +55,29 @@ function normalizedResourceHosts(value) {
     }
     return hosts;
 }
+function applicationMarkup(markup, resourceHosts, effectiveBaseUrl, documentOrigin) {
+    let resolutionBase;
+    try {
+        resolutionBase = new URL(typeof effectiveBaseUrl === 'string' ? effectiveBaseUrl : 'https://document.invalid/');
+    }
+    catch {
+        resolutionBase = new URL('https://document.invalid/');
+    }
+    const comparedOrigin = typeof documentOrigin === 'string' && /^https?:\/\//iu.test(documentOrigin)
+        ? documentOrigin
+        : null;
+    return markup.replace(/\s(src|href|poster|data)="([^"]*)"/giu, (attribute, name, value) => {
+        try {
+            const parsed = new URL(value, resolutionBase);
+            const knownExternalHost = resourceHosts.has(parsed.hostname.toLowerCase());
+            const offOrigin = comparedOrigin ? parsed.origin !== comparedOrigin : knownExternalHost;
+            return knownExternalHost && offOrigin ? ` ${name.toLowerCase()}=""` : attribute;
+        }
+        catch {
+            return attribute;
+        }
+    });
+}
 const PASSIVE_TECHNOLOGY_HEADER_NAMES = Object.freeze([
     'cf-ray',
     'x-drupal-cache',
@@ -106,6 +146,17 @@ function htmlEvidence(markers, description, confidence = 'high') {
         matches: ({ html }) => markers.some((marker) => html.includes(marker)),
     };
 }
+function resourcePathEvidence(markers, description, confidence = 'high') {
+    return {
+        source: 'static HTML',
+        description,
+        confidence,
+        matches: ({ html }) => markers.some((marker) => html.includes(marker)),
+        roleFor: ({ applicationHtml }) => markers.some((marker) => applicationHtml.includes(marker))
+            ? undefined
+            : 'embedded_dependency',
+    };
+}
 function resourceEvidence(hosts, description) {
     return {
         source: 'resource origin',
@@ -142,7 +193,10 @@ function responseHeaderEvidence(name, pattern, description, confidence = 'high')
         },
     };
 }
-function evidenceRole(signature, evidence) {
+function evidenceRole(signature, evidence, context) {
+    const contextualRole = context ? evidence.roleFor?.(context) : undefined;
+    if (contextualRole)
+        return contextualRole;
     if (evidence.role)
         return evidence.role;
     if (evidence.source === 'resource origin')
@@ -159,7 +213,7 @@ const TECHNOLOGY_SIGNATURES = [
         id: 'wordpress', name: 'WordPress', category: 'content management',
         evidence: [
             generatorEvidence(/^wordpress(?:\s|$)/i, 'Generator metadata identifies WordPress.'),
-            htmlEvidence(['/wp-content/', '/wp-includes/'], 'Static resource paths use WordPress conventions.', 'medium'),
+            resourcePathEvidence(['/wp-content/', '/wp-includes/'], 'Static resource paths use WordPress conventions.', 'medium'),
         ],
     },
     {
@@ -222,8 +276,9 @@ const TECHNOLOGY_SIGNATURES = [
     {
         id: 'bigcommerce', name: 'BigCommerce', category: 'commerce',
         minimumEvidenceMatches: 2,
+        allowEmbeddedOnly: true,
         evidence: [
-            htmlEvidence(['cdn11.bigcommerce.com/s-', 'stencil-utils'], 'Static markup contains BigCommerce storefront asset markers.', 'medium'),
+            resourcePathEvidence(['cdn11.bigcommerce.com/s-', 'stencil-utils'], 'Static markup contains BigCommerce storefront asset markers.', 'medium'),
             resourcePatternEvidence(/^cdn\d+\.bigcommerce\.com$/i, 'A retained resource origin uses BigCommerce storefront delivery infrastructure.'),
         ],
     },
@@ -421,28 +476,41 @@ function analyzeWebsiteTechnology(input = {}) {
     }) : null;
     const context = {
         html: htmlAnalysis.markup,
+        applicationHtml: '',
         generator: boundedLowercase(input.generator, MAX_GENERATOR_INPUT),
         httpServer: boundedLowercase(input.httpServer, MAX_SERVER_INPUT),
         resourceHosts: normalizedResourceHosts(input.resourceOrigins),
         responseHeaders: normalizedResponseHeaders(input.responseHeaders),
     };
+    context.applicationHtml = applicationMarkup(context.html, context.resourceHosts, input.effectiveBaseUrl, input.documentOrigin);
     const findings = [];
     for (const signature of TECHNOLOGY_SIGNATURES) {
-        const matched = signature.evidence.filter((evidence) => evidence.matches(context));
-        if (matched.length < (signature.minimumEvidenceMatches || 1))
-            continue;
-        if (signature.requiresNonResourceEvidence && matched.every((evidence) => evidence.source === 'resource origin'))
+        const matched = signature.evidence
+            .filter((evidence) => evidence.matches(context))
+            .map((evidence) => ({ evidence, role: evidenceRole(signature, evidence, context) }));
+        const nonEmbeddedMatches = matched.filter((entry) => entry.role !== 'embedded_dependency');
+        const minimumEvidenceMatches = signature.minimumEvidenceMatches || 1;
+        const qualifiesForPlatform = matched.length >= minimumEvidenceMatches
+            && (minimumEvidenceMatches === 1 || nonEmbeddedMatches.length > 0)
+            && (!signature.requiresNonResourceEvidence || nonEmbeddedMatches.length > 0);
+        const contextualEmbeddedEvidence = matched.filter((entry) => (entry.role === 'embedded_dependency' && entry.evidence.roleFor !== undefined));
+        const selected = qualifiesForPlatform
+            ? matched
+            : signature.allowEmbeddedOnly
+                ? contextualEmbeddedEvidence.slice(0, 1)
+                : [];
+        if (selected.length === 0)
             continue;
-        const matchedEvidence = matched.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map((evidence) => ({
+        const matchedEvidence = selected.slice(0, MAX_EVIDENCE_PER_TECHNOLOGY).map(({ evidence, role }) => ({
             source: evidence.source,
-            role: evidenceRole(signature, evidence),
+            role,
             description: evidence.description,
         }));
         findings.push({
             id: signature.id,
             name: signature.name,
             category: signature.category,
-            confidence: matched.some((evidence) => evidence.confidence === 'high') ? 'high' : 'medium',
+            confidence: selected.some(({ evidence }) => evidence.confidence === 'high') ? 'high' : 'medium',
             roles: TECHNOLOGY_EVIDENCE_ROLE_ORDER.filter((role) => matchedEvidence.some((evidence) => evidence.role === role)),
             evidence: matchedEvidence,
         });
@@ -493,4 +561,4 @@ function analyzeWebsiteTechnology(input = {}) {
         browserLibraryProfile,
     };
 }
-export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, captureTechnologyResponseHeaders, };
+export { MAX_EVIDENCE_PER_TECHNOLOGY, MAX_TECHNOLOGY_EVIDENCE_DESCRIPTION_LENGTH, MAX_TECHNOLOGY_FINDINGS, MAX_TECHNOLOGY_HTML_CHARS, MAX_TECHNOLOGY_TAGS, PASSIVE_TECHNOLOGY_HEADER_NAMES, TECHNOLOGY_CATEGORIES, TECHNOLOGY_EVIDENCE_SOURCES, TECHNOLOGY_PROFILE_VERSION, TECHNOLOGY_SIGNATURE_CATALOGUE, analyzeWebsiteTechnology, captureTechnologyResponseHeaders, };

package/lib/zone-intent-review.mjs

@@ -142,7 +142,11 @@ export function normaliseRdata(type, rawValue, origin, nameMode = 'absolute') {
         const family = type === 'A' ? 4 : 6;
         if (tokens.length !== 1 || isIP(tokens[0] ?? '') !== family)
             throw new TypeError(`${type} data must contain one valid address.`);
-        return { value: tokens[0].toLowerCase(), valueTreatment: 'normalised' };
+        const supplied = tokens[0];
+        const normalised = family === 4
+            ? supplied.split('.').map((part) => String(Number(part))).join('.')
+            : new URL(`http://[${supplied}]/`).hostname.slice(1, -1).toLowerCase();
+        return { value: normalised, valueTreatment: 'normalised' };
     }
     if (type === 'CNAME' || type === 'NS') {
         if (tokens.length !== 1)
@@ -152,7 +156,12 @@ export function normaliseRdata(type, rawValue, origin, nameMode = 'absolute') {
     if (type === 'MX') {
         if (tokens.length !== 2)
             throw new TypeError('MX data must contain preference and exchange.');
-        return { value: `${integerToken(tokens[0], 0, 65_535, 'MX preference')} ${host(tokens[1], 'MX exchange')}`, valueTreatment: 'normalised' };
+        const preference = integerToken(tokens[0], 0, 65_535, 'MX preference');
+        const exchangeToken = tokens[1];
+        if (exchangeToken === '.' && preference !== 0)
+            throw new TypeError('A Null MX exchange must use preference 0.');
+        const exchange = exchangeToken === '.' ? '.' : host(exchangeToken, 'MX exchange');
+        return { value: `${preference} ${exchange}`, valueTreatment: 'normalised' };
     }
     if (type === 'CAA') {
         if (tokens.length < 3)
@@ -380,7 +389,6 @@ function parseBindZone(zoneText, initialOrigin) {
                 if (tokens.length !== 2)
                     throw new TypeError('$ORIGIN requires one domain name.');
                 origin = domainName(tokens[1], origin, '$ORIGIN', 'master-file');
-                lastOwner = origin;
                 continue;
             }
             if (directive === '$TTL')
@@ -388,10 +396,12 @@ function parseBindZone(zoneText, initialOrigin) {
             if (directive?.startsWith('$'))
                 throw new TypeError(`${directive} is deliberately unsupported.`);
             let offset = 0;
-            const ownerToken = entry.ownerOmitted ? lastOwner : tokens[offset++];
-            if (!ownerToken)
+            const ownerToken = entry.ownerOmitted ? null : tokens[offset++];
+            if (!entry.ownerOmitted && !ownerToken)
                 throw new TypeError('Record owner is missing.');
-            const owner = ownerName(ownerToken, origin, 'record owner', 'master-file');
+            const owner = entry.ownerOmitted
+                ? lastOwner
+                : ownerName(ownerToken, origin, 'record owner', 'master-file');
             lastOwner = owner;
             let ttl = null;
             if (/^\d+$/u.test(tokens[offset] ?? ''))

package/package.json

@@ -33,7 +33,7 @@
   "bugs": {
     "url": "https://github.com/slicedearth/whoisleuth/issues"
   },
-  "version": "2.1.0",
+  "version": "2.3.0",
   "dependencies": {
     "@peculiar/x509": "2.0.0",
     "maxmind": "5.0.7",