@slicedearth/whoisleuth-cli 1.47.4 → 2.0.0

package/lib/static-html-analysis.mjs

@@ -4,6 +4,7 @@
 // Only normalized start-tag markup and capped script indicators are retained.
 import { Tokenizer, TokenizerMode } from 'parse5';
 import { MAX_CSP_META_POLICIES, MAX_RESPONSE_POLICY_HEADER_BYTES, } from "./response-policy.mjs";
+import { MAX_PAGE_PUBLICATION_DECLARATIONS, MAX_PAGE_PUBLICATION_META_ELEMENTS, MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES, } from "./homepage-metadata-contract.mjs";
 const MAX_STATIC_HTML_CHARS = 300_000;
 const MAX_STATIC_HTML_TAGS = 8_192;
 const MAX_STATIC_STRUCTURE_TOKENS = 4_096;
@@ -18,8 +19,12 @@ const MAX_INLINE_SCRIPT_TOTAL_CHARS = 65_536;
 const MAX_STATIC_VISIBLE_TEXT_CHARS = MAX_STATIC_HTML_CHARS;
 const MAX_STATIC_FORMS = 50;
 const MAX_STATIC_INPUTS = 500;
+const MAX_STATIC_PUBLICATION_META_ELEMENTS = MAX_PAGE_PUBLICATION_META_ELEMENTS;
+const MAX_STATIC_PUBLICATION_DECLARATIONS = MAX_PAGE_PUBLICATION_DECLARATIONS;
+const MAX_STATIC_ROBOTS_DIRECTIVES = MAX_PAGE_PUBLICATION_ROBOTS_DIRECTIVES;
 const MAX_FORM_ATTRIBUTE_LENGTH = 2_048;
-const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f]/;
+const CONTROL_CHARACTER_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
+const CONTROL_CHARACTER_RE_GLOBAL = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
 const PAYMENT_AUTOCOMPLETE_TOKENS = new Set([
     'cc-name',
     'cc-given-name',
@@ -50,6 +55,24 @@ const VOID_TAGS = new Set([
     'param', 'source', 'track', 'wbr',
 ]);
 const NON_VISIBLE_TEXT_TAGS = new Set(['script', 'style', 'template']);
+const HEAD_CONTEXT_TAGS = new Set([
+    'html', 'head', 'base', 'basefont', 'bgsound', 'link', 'meta', 'title',
+    'noscript', 'noframes', 'style', 'template', 'script',
+]);
+const IMPLICIT_HEAD_TEXT_TAGS = new Set([
+    'title', 'noscript', 'noframes', 'style', 'template', 'script',
+]);
+const ROBOTS_DIRECTIVES = new Set([
+    'all', 'follow', 'index', 'max-image-preview', 'max-snippet', 'max-video-preview',
+    'noarchive', 'nocache', 'nofollow', 'noimageindex', 'noindex', 'none', 'nositelinkssearchbox',
+    'nosnippet', 'notranslate', 'unavailable_after',
+]);
+const JAVASCRIPT_MEDIA_TYPES = new Set([
+    'application/ecmascript', 'application/javascript', 'application/x-ecmascript',
+    'application/x-javascript', 'text/ecmascript', 'text/javascript', 'text/javascript1.0',
+    'text/javascript1.1', 'text/javascript1.2', 'text/javascript1.3', 'text/javascript1.4',
+    'text/javascript1.5', 'text/jscript', 'text/livescript', 'text/x-ecmascript', 'text/x-javascript',
+]);
 function attributeValue(attributes, name, maximumLength = MAX_FORM_ATTRIBUTE_LENGTH) {
     const attribute = attributes
         .slice(0, MAX_ATTRIBUTES_PER_TAG)
@@ -171,6 +194,30 @@ function scriptMediaType(attributes) {
     }
     return null;
 }
+function classicScriptCandidate(attributes) {
+    const type = attributeValue(attributes, 'type', MAX_SCRIPT_MEDIA_TYPE_LENGTH);
+    const asyncAttribute = attributeValue(attributes, 'async', 10);
+    const deferAttribute = attributeValue(attributes, 'defer', 10);
+    const truncated = type.truncated || asyncAttribute.truncated || deferAttribute.truncated;
+    if (asyncAttribute.present || deferAttribute.present || type.value === 'module')
+        return { candidate: false, truncated };
+    return {
+        candidate: !type.present || type.value === '' || Boolean(type.value && JAVASCRIPT_MEDIA_TYPES.has(type.value)),
+        truncated,
+    };
+}
+function stylesheetCandidate(attributes) {
+    const rel = attributeValue(attributes, 'rel', 320);
+    const href = attributeValue(attributes, 'href', MAX_SCRIPT_REFERENCE_LENGTH);
+    const disabled = attributeValue(attributes, 'disabled', 10);
+    const media = attributeValue(attributes, 'media', 120);
+    return {
+        candidate: rel.value?.split(/\s+/u).includes('stylesheet') === true
+            && href.present && Boolean(href.value) && !disabled.present
+            && (!media.present || media.value === '' || media.value === 'all' || media.value === 'screen'),
+        truncated: rel.truncated || href.truncated || disabled.truncated || media.truncated,
+    };
+}
 function serializedStartTag(tagName, attributes) {
     let serialized = `<${tagName.toLowerCase()}`;
     let limitReached = attributes.length > MAX_ATTRIBUTES_PER_TAG;
@@ -207,6 +254,48 @@ function analyzeStaticHtml(value, options = {}) {
         actions: { sameOrigin: 0, external: 0, missing: 0, cleartext: 0, unclassified: 0 },
         truncated: false,
     };
+    const publicationMetadata = {
+        truncated: false,
+        documentTruncated: false,
+        robots: {
+            observed: false,
+            malformed: false,
+            truncated: false,
+            directives: [],
+            recognizedDirectiveCount: 0,
+            unknownDirectiveCount: 0,
+            conflicting: false,
+        },
+        twitterCard: {
+            observed: false,
+            malformed: false,
+            truncated: false,
+            cardTypes: [],
+            declarationCount: 0,
+            titlePresent: false,
+            descriptionPresent: false,
+            imagePresent: false,
+            imageAltPresent: false,
+            sitePresent: false,
+            creatorPresent: false,
+            playerPresent: false,
+            appPresent: false,
+        },
+        headings: { truncated: false, total: 0, h1: 0, h2: 0, h3: 0, h4: 0, h5: 0, h6: 0 },
+        images: {
+            totalComplete: true,
+            classificationComplete: true,
+            truncated: false,
+            total: 0,
+            altMissing: 0,
+            altEmpty: 0,
+            altNonEmpty: 0,
+            altUnclassified: 0,
+        },
+        renderBlockingCandidates: { truncated: false, script: 0, stylesheet: 0, total: 0 },
+    };
+    const robotDirectives = new Set();
+    const twitterCardTypes = new Set();
     let tokenizer;
     let activeInlineScript = null;
     let tagsExamined = 0;
@@ -217,7 +306,13 @@ function analyzeStaticHtml(value, options = {}) {
     let inlineLimitReached = false;
     let visibleTextLimitReached = false;
     let cspMetaLimitReached = false;
+    let publicationMetaElements = 0;
+    let robotDirectiveTokens = 0;
+    let publicationTagLimitReached = false;
     let insideExplicitHead = false;
+    let implicitHeadTextDepth = 0;
+    let headScopeClosed = false;
+    let bodyContentStarted = false;
     let scriptElementSeen = false;
     let nonVisibleTextDepth = 0;
     let visibleTextCharacters = 0;
@@ -263,6 +358,7 @@ function analyzeStaticHtml(value, options = {}) {
                 nonVisibleTextDepth += 1;
             if (tagsExamined >= MAX_STATIC_HTML_TAGS) {
                 tagLimitReached = true;
+                publicationTagLimitReached = true;
                 activeInlineScript = null;
                 return;
             }
@@ -270,6 +366,54 @@ function analyzeStaticHtml(value, options = {}) {
             appendStructureToken(token.selfClosing || VOID_TAGS.has(tagName) ? `${tagName}/` : tagName);
             if (tagName === 'head')
                 insideExplicitHead = true;
+            if (tagName === 'body' || (!insideExplicitHead && !HEAD_CONTEXT_TAGS.has(tagName))) {
+                bodyContentStarted = true;
+                headScopeClosed = true;
+            }
+            if (!insideExplicitHead && !headScopeClosed && !token.selfClosing && IMPLICIT_HEAD_TEXT_TAGS.has(tagName)) {
+                implicitHeadTextDepth += 1;
+            }
+            const inPublicationHead = insideExplicitHead || (!headScopeClosed && !bodyContentStarted && HEAD_CONTEXT_TAGS.has(tagName));
+            if (/^h[1-6]$/u.test(tagName)) {
+                const key = tagName;
+                publicationMetadata.headings[key] += 1;
+                publicationMetadata.headings.total += 1;
+            }
+            if (tagName === 'img') {
+                publicationMetadata.images.total += 1;
+                const alt = attributeValue(token.attrs, 'alt', MAX_FORM_ATTRIBUTE_LENGTH);
+                if (alt.truncated) {
+                    publicationMetadata.truncated = true;
+                    publicationMetadata.images.classificationComplete = false;
+                    publicationMetadata.images.truncated = true;
+                }
+                if (alt.truncated && (!alt.present || alt.value === null))
+                    publicationMetadata.images.altUnclassified += 1;
+                else if (!alt.present)
+                    publicationMetadata.images.altMissing += 1;
+                else if (alt.value === '')
+                    publicationMetadata.images.altEmpty += 1;
+                else if (alt.value !== null)
+                    publicationMetadata.images.altNonEmpty += 1;
+            }
+            if (insideExplicitHead && tagName === 'script') {
+                const candidate = classicScriptCandidate(token.attrs);
+                if (candidate.truncated) {
+                    publicationMetadata.truncated = true;
+                    publicationMetadata.renderBlockingCandidates.truncated = true;
+                }
+                if (candidate.candidate)
+                    publicationMetadata.renderBlockingCandidates.script += 1;
+            }
+            else if (insideExplicitHead && tagName === 'link') {
+                const candidate = stylesheetCandidate(token.attrs);
+                if (candidate.truncated) {
+                    publicationMetadata.truncated = true;
+                    publicationMetadata.renderBlockingCandidates.truncated = true;
+                }
+                if (candidate.candidate)
+                    publicationMetadata.renderBlockingCandidates.stylesheet += 1;
+            }
             if (tagName === 'meta' && insideExplicitHead) {
                 const httpEquiv = attributeValue(token.attrs, 'http-equiv', 64);
                 if (httpEquiv.value === 'content-security-policy') {
@@ -285,6 +429,125 @@ function analyzeStaticHtml(value, options = {}) {
                     }
                 }
             }
+            if (tagName === 'meta' && inPublicationHead) {
+                const name = attributeValue(token.attrs, 'name', 120);
+                const property = attributeValue(token.attrs, 'property', 120);
+                const suppliedKeys = [name.value, property.value].filter((value) => Boolean(value));
+                const key = suppliedKeys[0] || null;
+                const conflictingKeys = suppliedKeys.length > 1 && new Set(suppliedKeys).size > 1;
+                if (name.truncated || property.truncated) {
+                    publicationMetadata.truncated = true;
+                    // An over-bound or control-bearing key can hide which declaration
+                    // family it belongs to. Mark both bounded declaration families
+                    // partial instead of emitting a root state its children contradict.
+                    publicationMetadata.robots.truncated = true;
+                    publicationMetadata.twitterCard.truncated = true;
+                }
+                const publicationKey = suppliedKeys.some((value) => value === 'robots' || value.startsWith('twitter:'));
+                const processPublication = publicationKey
+                    && publicationMetaElements < MAX_STATIC_PUBLICATION_META_ELEMENTS;
+                if (publicationKey && !processPublication) {
+                    publicationMetadata.truncated = true;
+                    if (suppliedKeys.includes('robots'))
+                        publicationMetadata.robots.truncated = true;
+                    if (suppliedKeys.some((value) => value.startsWith('twitter:')))
+                        publicationMetadata.twitterCard.truncated = true;
+                }
+                else if (processPublication) {
+                    publicationMetaElements += 1;
+                }
+                if (processPublication && conflictingKeys) {
+                    if (suppliedKeys.includes('robots')) {
+                        publicationMetadata.robots.observed = true;
+                        publicationMetadata.robots.malformed = true;
+                    }
+                    if (suppliedKeys.some((value) => value.startsWith('twitter:'))) {
+                        publicationMetadata.twitterCard.observed = true;
+                        publicationMetadata.twitterCard.malformed = true;
+                    }
+                }
+                else if (processPublication && key === 'robots') {
+                    publicationMetadata.robots.observed = true;
+                    const content = attributeValue(token.attrs, 'content', 1_024);
+                    if (content.value === null || content.truncated) {
+                        publicationMetadata.robots.malformed = true;
+                        publicationMetadata.truncated = publicationMetadata.truncated || content.truncated;
+                        publicationMetadata.robots.truncated = publicationMetadata.robots.truncated || content.truncated;
+                    }
+                    else {
+                        for (const suppliedDirective of content.value.split(/[,\t\n\f\r ]+/u)) {
+                            if (robotDirectiveTokens >= MAX_STATIC_ROBOTS_DIRECTIVES) {
+                                publicationMetadata.truncated = true;
+                                publicationMetadata.robots.truncated = true;
+                                break;
+                            }
+                            robotDirectiveTokens += 1;
+                            const directive = suppliedDirective.trim();
+                            if (!directive) {
+                                publicationMetadata.robots.malformed = true;
+                                continue;
+                            }
+                            const directiveName = directive.split(':', 1)[0].trim().replace(/\s+/gu, '_');
+                            if (ROBOTS_DIRECTIVES.has(directiveName)) {
+                                publicationMetadata.robots.recognizedDirectiveCount += 1;
+                                robotDirectives.add(directiveName);
+                            }
+                            else {
+                                publicationMetadata.robots.unknownDirectiveCount += 1;
+                            }
+                        }
+                    }
+                }
+                else if (processPublication && key?.startsWith('twitter:')) {
+                    const category = key === 'twitter:card' ? 'card'
+                        : key === 'twitter:title' ? 'title'
+                            : key === 'twitter:description' ? 'description'
+                                : key === 'twitter:image:alt' ? 'imageAlt'
+                                    : key === 'twitter:image' || key.startsWith('twitter:image:') ? 'image'
+                                        : key === 'twitter:site' || key.startsWith('twitter:site:') ? 'site'
+                                            : key === 'twitter:creator' || key.startsWith('twitter:creator:') ? 'creator'
+                                                : key === 'twitter:player' || key.startsWith('twitter:player:') ? 'player'
+                                                    : key === 'twitter:app' || key.startsWith('twitter:app:') ? 'app'
+                                                        : null;
+                    if (category) {
+                        publicationMetadata.twitterCard.observed = true;
+                        if (publicationMetadata.twitterCard.declarationCount >= MAX_STATIC_PUBLICATION_DECLARATIONS) {
+                            publicationMetadata.truncated = true;
+                            publicationMetadata.twitterCard.truncated = true;
+                        }
+                        else {
+                            publicationMetadata.twitterCard.declarationCount += 1;
+                            const content = attributeValue(token.attrs, 'content', 1_024);
+                            if (content.value === null || content.truncated || !content.value) {
+                                publicationMetadata.twitterCard.malformed = true;
+                                publicationMetadata.truncated = publicationMetadata.truncated || content.truncated;
+                                publicationMetadata.twitterCard.truncated = publicationMetadata.twitterCard.truncated || content.truncated;
+                            }
+                            else if (category === 'card') {
+                                twitterCardTypes.add(['summary', 'summary_large_image', 'player', 'app'].includes(content.value)
+                                    ? content.value
+                                    : 'other');
+                            }
+                            else if (category === 'title')
+                                publicationMetadata.twitterCard.titlePresent = true;
+                            else if (category === 'description')
+                                publicationMetadata.twitterCard.descriptionPresent = true;
+                            else if (category === 'image')
+                                publicationMetadata.twitterCard.imagePresent = true;
+                            else if (category === 'imageAlt')
+                                publicationMetadata.twitterCard.imageAltPresent = true;
+                            else if (category === 'site')
+                                publicationMetadata.twitterCard.sitePresent = true;
+                            else if (category === 'creator')
+                                publicationMetadata.twitterCard.creatorPresent = true;
+                            else if (category === 'player')
+                                publicationMetadata.twitterCard.playerPresent = true;
+                            else if (category === 'app')
+                                publicationMetadata.twitterCard.appPresent = true;
+                        }
+                    }
+                }
+            }
             if (markup.length < MAX_TECHNOLOGY_TAGS) {
                 const serialized = serializedStartTag(tagName, token.attrs);
                 if (serialized.limitReached)
@@ -349,18 +612,31 @@ function analyzeStaticHtml(value, options = {}) {
             const tagName = token.tagName.toLowerCase();
             if (tagName === 'script')
                 activeInlineScript = null;
-            if (tagName === 'head')
+            if (tagName === 'head') {
                 insideExplicitHead = false;
+                headScopeClosed = true;
+            }
+            if (!insideExplicitHead && IMPLICIT_HEAD_TEXT_TAGS.has(tagName) && implicitHeadTextDepth > 0) {
+                implicitHeadTextDepth -= 1;
+            }
             if (!VOID_TAGS.has(tagName))
                 appendStructureToken(`/${tagName}`);
             if (NON_VISIBLE_TEXT_TAGS.has(tagName) && nonVisibleTextDepth > 0)
                 nonVisibleTextDepth -= 1;
         },
         onCharacter(token) {
+            if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0 && token.chars.trim()) {
+                bodyContentStarted = true;
+                headScopeClosed = true;
+            }
             appendInlineScript(token.chars);
             appendVisibleText(token.chars);
         },
         onNullCharacter(token) {
+            if (!insideExplicitHead && !headScopeClosed && implicitHeadTextDepth === 0) {
+                bodyContentStarted = true;
+                headScopeClosed = true;
+            }
             appendInlineScript(token.chars);
             appendVisibleText(token.chars);
         },
@@ -374,14 +650,35 @@ function analyzeStaticHtml(value, options = {}) {
     };
     tokenizer = new Tokenizer({ sourceCodeLocationInfo: false }, handler);
     tokenizer.write(html, true);
+    publicationMetadata.truncated = publicationMetadata.truncated || inputLimitReached || publicationTagLimitReached;
+    publicationMetadata.documentTruncated = inputLimitReached || publicationTagLimitReached;
+    if (publicationMetadata.documentTruncated) {
+        publicationMetadata.robots.truncated = true;
+        publicationMetadata.twitterCard.truncated = true;
+        publicationMetadata.headings.truncated = true;
+        publicationMetadata.images.totalComplete = false;
+        publicationMetadata.images.classificationComplete = false;
+        publicationMetadata.images.truncated = true;
+        publicationMetadata.renderBlockingCandidates.truncated = true;
+    }
+    publicationMetadata.robots.directives = [...robotDirectives].sort();
+    publicationMetadata.robots.conflicting = ((robotDirectives.has('index') || robotDirectives.has('all'))
+        && (robotDirectives.has('noindex') || robotDirectives.has('none'))) || ((robotDirectives.has('follow') || robotDirectives.has('all'))
+        && (robotDirectives.has('nofollow') || robotDirectives.has('none')));
+    publicationMetadata.twitterCard.cardTypes = [...twitterCardTypes].sort().slice(0, 8);
+    if (twitterCardTypes.size > 8)
+        publicationMetadata.truncated = true;
+    publicationMetadata.renderBlockingCandidates.total = publicationMetadata.renderBlockingCandidates.script
+        + publicationMetadata.renderBlockingCandidates.stylesheet;
     return {
         markup: markup.join('\n'),
-        visibleText: visibleTextParts.join(''),
+        visibleText: visibleTextParts.join('').replace(CONTROL_CHARACTER_RE_GLOBAL, ' ').replace(/\s+/gu, ' '),
         structureTokens,
         scripts,
         cspMetaPolicies,
         cspMetaLimitReached,
         forms,
+        publicationMetadata,
         inputLimitReached,
         tagLimitReached,
         structureLimitReached,
@@ -392,4 +689,4 @@ function analyzeStaticHtml(value, options = {}) {
         inlineCharactersExamined,
     };
 }
-export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };
+export { MAX_ATTRIBUTES_PER_TAG, MAX_INLINE_SCRIPT_CHARS, MAX_INLINE_SCRIPT_TOTAL_CHARS, MAX_SCRIPT_ELEMENTS, MAX_SCRIPT_MEDIA_TYPE_LENGTH, MAX_STATIC_FORMS, MAX_STATIC_HTML_CHARS, MAX_STATIC_INPUTS, MAX_STATIC_PUBLICATION_META_ELEMENTS, MAX_STATIC_PUBLICATION_DECLARATIONS, MAX_STATIC_ROBOTS_DIRECTIVES, MAX_STATIC_HTML_TAGS, MAX_STATIC_STRUCTURE_TOKENS, MAX_STATIC_VISIBLE_TEXT_CHARS, MAX_TAG_LENGTH, MAX_TECHNOLOGY_TAGS, analyzeStaticHtml, };

package/lib/structured-data-identity.mjs

@@ -4,7 +4,8 @@
 // and fixed limitations. Raw JSON-LD and arbitrary publisher properties are
 // discarded immediately after analysis.
 import { domainToASCII } from 'node:url';
-import { createObservation } from "./observation.mjs";
+import { createObservation } from "../packages/evidence/observation.mjs";
+import { isUriShapedLabel } from "./portable-generator.mjs";
 import { analyzeStaticHtml, } from "./static-html-analysis.mjs";
 const STRUCTURED_DATA_IDENTITY_VERSION = 1;
 const MAX_STRUCTURED_DATA_SCRIPTS = 8;
@@ -19,8 +20,8 @@ const MAX_STRUCTURED_DATA_TYPES = 8;
 const MAX_STRUCTURED_DATA_NAME_LENGTH = 160;
 const MAX_STRUCTURED_DATA_URL_LENGTH = 2_048;
 const MAX_STRUCTURED_DATA_SAME_AS_HOSTS = 12;
-const CONTROL_AND_DIRECTIONAL_RE = /[\u0000-\u001f\u007f\u061c\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/u;
-const CONTROL_AND_DIRECTIONAL_GLOBAL_RE = /[\u0000-\u001f\u007f\u061c\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/gu;
+const CONTROL_AND_DIRECTIONAL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/u;
+const CONTROL_AND_DIRECTIONAL_GLOBAL_RE = /[\u0000-\u001f\u007f-\u009f]|\p{Default_Ignorable_Code_Point}/gu;
 const HOSTNAME_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/iu;
 const CURATED_TYPES = new Map([
     ['brand', 'Brand'],
@@ -44,17 +45,21 @@ function record(value) {
         : null;
 }
 function boundedName(value) {
-    if (typeof value !== 'string' || value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
-        return null;
+    if (typeof value !== 'string')
+        return { value: null, truncated: false };
+    if (value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
+        return { value: null, truncated: true };
+    if (isUriShapedLabel(value))
+        return { value: null, truncated: false };
     const normalized = value
         .replace(CONTROL_AND_DIRECTIONAL_GLOBAL_RE, ' ')
         .replace(/\s+/gu, ' ')
         .trim();
     if (!normalized)
-        return null;
+        return { value: null, truncated: false };
     return normalized.length <= MAX_STRUCTURED_DATA_NAME_LENGTH
-        ? normalized
-        : `${normalized.slice(0, MAX_STRUCTURED_DATA_NAME_LENGTH)}…`;
+        ? { value: normalized, truncated: false }
+        : { value: `${normalized.slice(0, MAX_STRUCTURED_DATA_NAME_LENGTH - 1)}…`, truncated: true };
 }
 function safeBaseUrl(value) {
     if (typeof value !== 'string' || value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
@@ -116,7 +121,7 @@ function curatedTypes(value) {
         if (normalized)
             types.add(normalized);
     }
-    return [...types].sort();
+    return { value: [...types].sort(), truncated: values.length > MAX_STRUCTURED_DATA_TYPES };
 }
 function sameAsHosts(value) {
     const values = Array.isArray(value) ? value : [value];
@@ -126,7 +131,7 @@ function sameAsHosts(value) {
         if (host)
             hosts.add(host);
     }
-    return [...hosts].sort();
+    return { value: [...hosts].sort(), truncated: values.length > MAX_STRUCTURED_DATA_SAME_AS_HOSTS };
 }
 function jsonStructureWithinBounds(value) {
     let depth = 0;
@@ -161,14 +166,21 @@ function jsonStructureWithinBounds(value) {
 }
 function entityFromObject(value, baseUrl) {
     const types = curatedTypes(value['@type']);
-    if (!types.length)
-        return null;
     const name = boundedName(value.name);
     const declaredOrigin = normalizedOrigin(value.url, baseUrl);
     const hosts = sameAsHosts(value.sameAs);
-    if (!name && !declaredOrigin && !hosts.length)
-        return null;
-    return { types, name, declaredOrigin, sameAsHosts: hosts };
+    const projection = {
+        nameTruncated: name.truncated,
+        typesTruncated: types.truncated,
+        sameAsTruncated: hosts.truncated,
+    };
+    if (!types.value.length || (!name.value && !declaredOrigin && !hosts.value.length)) {
+        return { entity: null, ...projection };
+    }
+    return {
+        entity: { types: types.value, name: name.value, declaredOrigin, sameAsHosts: hosts.value },
+        ...projection,
+    };
 }
 function analyzeStructuredDataIdentity(input = {}) {
     const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
@@ -183,6 +195,9 @@ function analyzeStructuredDataIdentity(input = {}) {
     let objectsExamined = 0;
     let arrayItemsExamined = 0;
     let discardedProperties = 0;
+    let truncatedNames = 0;
+    let truncatedTypeLists = 0;
+    let truncatedSameAsLists = 0;
     let entityLimitReached = false;
     let limitReached = scripts.length > MAX_STRUCTURED_DATA_SCRIPTS;
     for (const script of selectedScripts) {
@@ -242,12 +257,20 @@ function analyzeStructuredDataIdentity(input = {}) {
                 continue;
             }
             objectsExamined += 1;
-            const entity = entityFromObject(object, baseUrl);
-            if (entity) {
-                const key = JSON.stringify(entity);
+            const projection = entityFromObject(object, baseUrl);
+            if (projection.nameTruncated)
+                truncatedNames += 1;
+            if (projection.typesTruncated)
+                truncatedTypeLists += 1;
+            if (projection.sameAsTruncated)
+                truncatedSameAsLists += 1;
+            if (projection.nameTruncated || projection.typesTruncated || projection.sameAsTruncated)
+                limitReached = true;
+            if (projection.entity) {
+                const key = JSON.stringify(projection.entity);
                 if (!entities.has(key)) {
                     if (entities.size < MAX_STRUCTURED_DATA_ENTITIES)
-                        entities.set(key, entity);
+                        entities.set(key, projection.entity);
                     else {
                         entityLimitReached = true;
                         limitReached = true;
@@ -294,6 +317,12 @@ function analyzeStructuredDataIdentity(input = {}) {
     if (entityLimitReached) {
         limitations.push(`Only the first ${MAX_STRUCTURED_DATA_ENTITIES} structured identity entities were retained.`);
     }
+    if (truncatedNames > 0)
+        limitations.push(`${truncatedNames} structured identity label${truncatedNames === 1 ? ' was' : 's were'} shortened to the ${MAX_STRUCTURED_DATA_NAME_LENGTH}-character retention limit.`);
+    if (truncatedTypeLists > 0)
+        limitations.push(`${truncatedTypeLists} structured identity type list${truncatedTypeLists === 1 ? ' exceeded' : 's exceeded'} the ${MAX_STRUCTURED_DATA_TYPES}-item inspection limit.`);
+    if (truncatedSameAsLists > 0)
+        limitations.push(`${truncatedSameAsLists} structured identity sameAs list${truncatedSameAsLists === 1 ? ' exceeded' : 's exceeded'} the ${MAX_STRUCTURED_DATA_SAME_AS_HOSTS}-item inspection limit.`);
     return {
         structuredDataVersion: STRUCTURED_DATA_IDENTITY_VERSION,
         ...createObservation({
@@ -314,6 +343,9 @@ function analyzeStructuredDataIdentity(input = {}) {
                 objectsExamined,
                 arrayItemsExamined,
                 discardedProperties,
+                truncatedNames,
+                truncatedTypeLists,
+                truncatedSameAsLists,
                 entities: entities.size,
             },
         }),

package/lib/threat-intelligence-fixtures.mjs

@@ -1,5 +1,6 @@
 import { createCuratedConnectorResult, normalizeCuratedConnectorTarget, } from "./threat-intelligence-runtime.mjs";
 import { assertCuratedConnectorDefinition } from "./threat-intelligence-definition-registry.mjs";
+import { normalizeLegacyIsoTimestamp } from "../packages/evidence/observation.mjs";
 const MAX_CONNECTOR_KEY_LENGTH = 80;
 const MAX_CONNECTOR_FIXTURE_BYTES = 512 * 1024;
 function isRecord(value) {
@@ -24,10 +25,7 @@ function fixtureId(value) {
     return value;
 }
 function fixtureTimestamp(value) {
-    if (typeof value !== 'string' || value.length > 50)
-        return null;
-    const timestamp = Date.parse(value);
-    return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
+    return normalizeLegacyIsoTimestamp(value);
 }
 export function runCuratedConnectorFixture(connector, fixture, normalizeFixture) {
     assertCuratedConnectorDefinition(connector);

package/lib/threat-intelligence-manifests.mjs

@@ -1,5 +1,6 @@
 import { registerCuratedConnector, registerThreatIntelligenceProvider, isCuratedConnectorDefinition, isThreatIntelligenceProvider, } from "./threat-intelligence-definition-registry.mjs";
 import { CURATED_CONNECTOR_CONTRACT_VERSION, MAX_CACHE_TTL_MS, MAX_CONNECTOR_ENTITIES, MAX_CONNECTOR_RELATIONSHIPS, MAX_RESPONSE_BYTES, MAX_TIMEOUT_MS, THREAT_INTELLIGENCE_CONTRACT_VERSION, } from "./threat-intelligence-runtime.mjs";
+import { normalizeLegacyIsoTimestamp } from "../packages/evidence/observation.mjs";
 const MAX_PROVIDER_ID_LENGTH = 40;
 const MAX_PROVIDER_LABEL_LENGTH = 100;
 const MAX_URL_LENGTH = 2_048;
@@ -137,13 +138,7 @@ function strictBoundedString(value, maximum) {
     return normalized && normalized.length <= maximum ? normalized : null;
 }
 function isoTimestamp(value) {
-    if (typeof value !== 'string' ||
-        value.length > 64 ||
-        /[\u0000-\u001f\u007f]/u.test(value)) {
-        return null;
-    }
-    const timestamp = Date.parse(value);
-    return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
+    return normalizeLegacyIsoTimestamp(value);
 }
 function httpsUrl(value) {
     const raw = strictBoundedString(value, MAX_URL_LENGTH);

package/lib/threat-intelligence-runtime.mjs

@@ -3,13 +3,11 @@
 // network requests and contains no provider credentials. Adapters added later
 // must declare exact input exposure, reviewed terms, and bounded request and
 // output budgets before producing separately attributed normalized evidence.
-import { createObservation } from "./observation.mjs";
-import { THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_RESULT_STATES, } from "./threat-intelligence-types.mjs";
+import { createObservation } from "../packages/evidence/observation.mjs";
+import { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_CATEGORIES, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_RESULT_STATES, THREAT_INTELLIGENCE_SCHEMA, } from "./threat-intelligence-types.mjs";
 import { assertCuratedConnectorDefinition, assertThreatIntelligenceProvider, } from "./threat-intelligence-definition-registry.mjs";
 import { boundedHttpsUrl, boundedString, isRecord, isoTimestamp, strictBoundedString, } from "./bounded-contract-normalizers.mjs";
 import { CURATED_CONNECTOR_ENTITY_TYPES, MAX_THREAT_INTELLIGENCE_URL_LENGTH, normalizeCuratedConnectorTarget, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-targets.mjs";
-const THREAT_INTELLIGENCE_CONTRACT_VERSION = 1;
-const THREAT_INTELLIGENCE_SCHEMA = 'whoisleuth.threat-intelligence-result';
 const CURATED_CONNECTOR_CONTRACT_VERSION = 1;
 const CURATED_CONNECTOR_RESULT_SCHEMA = 'whoisleuth.curated-connector-result';
 const RESULT_STATES = new Set(THREAT_INTELLIGENCE_RESULT_STATES);
@@ -650,4 +648,4 @@ function createCuratedConnectorResult(connector, target, input = {}, observedAt
         }),
     };
 }
-export { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_SCHEMA, CURATED_CONNECTOR_CONTRACT_VERSION, CURATED_CONNECTOR_RESULT_SCHEMA, MAX_FINDINGS, MAX_INPUT_FINDINGS, MAX_TIMEOUT_MS, MAX_RESPONSE_BYTES, MAX_CACHE_TTL_MS, MAX_CONNECTOR_ENTITIES, MAX_CONNECTOR_RELATIONSHIPS, MAX_CONNECTOR_INPUT_ENTITIES, MAX_CONNECTOR_INPUT_RELATIONSHIPS, normalizeThreatIntelligenceTarget, normalizeCuratedConnectorTarget, createThreatIntelligenceResult, createCuratedConnectorResult, };
+export { THREAT_INTELLIGENCE_CONTRACT_VERSION, THREAT_INTELLIGENCE_ENVELOPE_VERSION, THREAT_INTELLIGENCE_SCHEMA, CURATED_CONNECTOR_CONTRACT_VERSION, CURATED_CONNECTOR_RESULT_SCHEMA, MAX_FINDINGS, MAX_INPUT_FINDINGS, MAX_TIMEOUT_MS, MAX_RESPONSE_BYTES, MAX_CACHE_TTL_MS, MAX_CONNECTOR_ENTITIES, MAX_CONNECTOR_RELATIONSHIPS, MAX_CONNECTOR_INPUT_ENTITIES, MAX_CONNECTOR_INPUT_RELATIONSHIPS, normalizeThreatIntelligenceTarget, normalizeCuratedConnectorTarget, createThreatIntelligenceResult, createCuratedConnectorResult, };

package/lib/threat-intelligence-types.mjs

@@ -1,5 +1,8 @@
 // Provider-neutral types shared by threat-intelligence and curated connector
 // manifests, fixture harnesses, normalizers, and result builders.
+export const THREAT_INTELLIGENCE_SCHEMA = 'whoisleuth.threat-intelligence-result';
+export const THREAT_INTELLIGENCE_CONTRACT_VERSION = 1;
+export const THREAT_INTELLIGENCE_ENVELOPE_VERSION = 1;
 export const THREAT_INTELLIGENCE_CATEGORIES = Object.freeze([
     'phishing',
     'malware',

package/lib/threatfox-intelligence.mjs

@@ -6,6 +6,7 @@ import { safeFetchDetailed, readTextCapped } from "./safe-fetch.mjs";
 import { whoisleuthRequestHeaders } from "./outbound-identity.mjs";
 import { providerPolicyAdmission } from "./provider-policy-admission.mjs";
 import { createThreatIntelligenceResult, defineThreatIntelligenceProvider, normalizeThreatIntelligenceTarget, } from "./threat-intelligence-contract.mjs";
+import { normalizeExplicitIsoTimestamp } from "../packages/evidence/observation.mjs";
 const THREATFOX_SEARCH_ENDPOINT = 'https://threatfox-api.abuse.ch/api/v1/';
 const THREATFOX_MAX_RESULTS = 20;
 const THREATFOX_MAX_INPUT_RESULTS = 100;
@@ -75,8 +76,7 @@ function isoTimestamp(value) {
     const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}(?: UTC)?$/u.test(raw)
         ? `${raw.slice(0, 10)}T${raw.slice(11, 19)}Z`
         : raw;
-    const timestamp = Date.parse(normalized);
-    return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
+    return normalizeExplicitIsoTimestamp(normalized);
 }
 function normalizedTags(value) {
     if (!Array.isArray(value))
@@ -112,7 +112,11 @@ function normalizeIocFinding(value, targetDomain) {
     if (!id || !/^\d{1,32}$/u.test(id) || ioc !== targetDomain || iocType !== 'domain' || !threatType)
         return null;
     const firstObservedAt = isoTimestamp(value.first_seen);
-    const lastObservedAt = isoTimestamp(value.last_seen) || firstObservedAt;
+    const parsedLastObservedAt = isoTimestamp(value.last_seen);
+    if ((value.first_seen !== undefined && value.first_seen !== null && !firstObservedAt)
+        || (value.last_seen !== undefined && value.last_seen !== null && !parsedLastObservedAt))
+        return null;
+    const lastObservedAt = parsedLastObservedAt || firstObservedAt;
     const tags = normalizedTags(value.tags);
     tags.push(`role:${threatType}`);
     if (malware)