@slicedearth/whoisleuth-cli 1.32.0

package/lib/structured-data-identity.mjs

@@ -0,0 +1,323 @@
+// Privacy-minimized identity claims derived from JSON-LD script elements in
+// the already-captured deep-Lookup homepage response. The bounded projection
+// retains only curated schema types, labels, origins, hostnames, source health,
+// and fixed limitations. Raw JSON-LD and arbitrary publisher properties are
+// discarded immediately after analysis.
+import { domainToASCII } from 'node:url';
+import { createObservation } from "./observation.mjs";
+import { analyzeStaticHtml, } from "./static-html-analysis.mjs";
+const STRUCTURED_DATA_IDENTITY_VERSION = 1;
+const MAX_STRUCTURED_DATA_SCRIPTS = 8;
+const MAX_STRUCTURED_DATA_SCRIPT_CHARS = 16_384;
+const MAX_STRUCTURED_DATA_TOTAL_CHARS = 32_768;
+const MAX_STRUCTURED_DATA_DEPTH = 16;
+const MAX_STRUCTURED_DATA_OBJECTS = 256;
+const MAX_STRUCTURED_DATA_ARRAY_ITEMS = 256;
+const MAX_STRUCTURED_DATA_PROPERTIES = 64;
+const MAX_STRUCTURED_DATA_ENTITIES = 16;
+const MAX_STRUCTURED_DATA_TYPES = 8;
+const MAX_STRUCTURED_DATA_NAME_LENGTH = 160;
+const MAX_STRUCTURED_DATA_URL_LENGTH = 2_048;
+const MAX_STRUCTURED_DATA_SAME_AS_HOSTS = 12;
+const CONTROL_AND_DIRECTIONAL_RE = /[\u0000-\u001f\u007f\u061c\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/u;
+const CONTROL_AND_DIRECTIONAL_GLOBAL_RE = /[\u0000-\u001f\u007f\u061c\u200b-\u200f\u202a-\u202e\u2060-\u2069\ufeff]/gu;
+const HOSTNAME_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/iu;
+const CURATED_TYPES = new Map([
+    ['brand', 'Brand'],
+    ['corporation', 'Corporation'],
+    ['educationalorganization', 'EducationalOrganization'],
+    ['governmentorganization', 'GovernmentOrganization'],
+    ['localbusiness', 'LocalBusiness'],
+    ['newsmediaorganization', 'NewsMediaOrganization'],
+    ['ngo', 'NGO'],
+    ['onlinebusiness', 'OnlineBusiness'],
+    ['organization', 'Organization'],
+    ['performinggroup', 'PerformingGroup'],
+    ['project', 'Project'],
+    ['sportsorganization', 'SportsOrganization'],
+    ['webpage', 'WebPage'],
+    ['website', 'WebSite'],
+]);
+function record(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : null;
+}
+function boundedName(value) {
+    if (typeof value !== 'string' || value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
+        return null;
+    const normalized = value
+        .replace(CONTROL_AND_DIRECTIONAL_GLOBAL_RE, ' ')
+        .replace(/\s+/gu, ' ')
+        .trim();
+    if (!normalized)
+        return null;
+    return normalized.length <= MAX_STRUCTURED_DATA_NAME_LENGTH
+        ? normalized
+        : `${normalized.slice(0, MAX_STRUCTURED_DATA_NAME_LENGTH)}…`;
+}
+function safeBaseUrl(value) {
+    if (typeof value !== 'string' || value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
+        return null;
+    try {
+        const parsed = new URL(value);
+        if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname) {
+            return null;
+        }
+        return parsed.toString();
+    }
+    catch {
+        return null;
+    }
+}
+function normalizedOrigin(value, baseUrl) {
+    if (typeof value !== 'string' || value.length === 0 || value.length > MAX_STRUCTURED_DATA_URL_LENGTH)
+        return null;
+    if (CONTROL_AND_DIRECTIONAL_RE.test(value))
+        return null;
+    try {
+        const parsed = baseUrl ? new URL(value, baseUrl) : new URL(value);
+        if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname)
+            return null;
+        return parsed.origin;
+    }
+    catch {
+        return null;
+    }
+}
+function normalizedSameAsHost(value) {
+    if (typeof value !== 'string'
+        || value.length === 0
+        || value.length > MAX_STRUCTURED_DATA_URL_LENGTH
+        || CONTROL_AND_DIRECTIONAL_RE.test(value))
+        return null;
+    try {
+        const parsed = new URL(value);
+        if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password || !parsed.hostname)
+            return null;
+        const ascii = domainToASCII(parsed.hostname.toLowerCase().replace(/\.$/u, ''));
+        return ascii && HOSTNAME_RE.test(ascii) ? ascii : null;
+    }
+    catch {
+        return null;
+    }
+}
+function curatedType(value) {
+    if (typeof value !== 'string' || value.length === 0 || value.length > 160)
+        return null;
+    const suffix = value.trim().split(/[\/#]/u).pop()?.replace(/[^a-z]/giu, '').toLowerCase() || '';
+    return CURATED_TYPES.get(suffix) || null;
+}
+function curatedTypes(value) {
+    const values = Array.isArray(value) ? value : [value];
+    const types = new Set();
+    for (const candidate of values.slice(0, MAX_STRUCTURED_DATA_TYPES)) {
+        const normalized = curatedType(candidate);
+        if (normalized)
+            types.add(normalized);
+    }
+    return [...types].sort();
+}
+function sameAsHosts(value) {
+    const values = Array.isArray(value) ? value : [value];
+    const hosts = new Set();
+    for (const candidate of values.slice(0, MAX_STRUCTURED_DATA_SAME_AS_HOSTS)) {
+        const host = normalizedSameAsHost(candidate);
+        if (host)
+            hosts.add(host);
+    }
+    return [...hosts].sort();
+}
+function jsonStructureWithinBounds(value) {
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (const character of value) {
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (character === '\\')
+                escaped = true;
+            else if (character === '"')
+                inString = false;
+            continue;
+        }
+        if (character === '"') {
+            inString = true;
+            continue;
+        }
+        if (character === '{' || character === '[') {
+            depth += 1;
+            if (depth > MAX_STRUCTURED_DATA_DEPTH)
+                return false;
+        }
+        else if (character === '}' || character === ']') {
+            depth -= 1;
+            if (depth < 0)
+                return false;
+        }
+    }
+    return !inString && depth === 0;
+}
+function entityFromObject(value, baseUrl) {
+    const types = curatedTypes(value['@type']);
+    if (!types.length)
+        return null;
+    const name = boundedName(value.name);
+    const declaredOrigin = normalizedOrigin(value.url, baseUrl);
+    const hosts = sameAsHosts(value.sameAs);
+    if (!name && !declaredOrigin && !hosts.length)
+        return null;
+    return { types, name, declaredOrigin, sameAsHosts: hosts };
+}
+function analyzeStructuredDataIdentity(input = {}) {
+    const htmlAnalysis = input.htmlAnalysis ?? analyzeStaticHtml(input.html);
+    const baseUrl = safeBaseUrl(input.baseUrl);
+    const scripts = htmlAnalysis.scripts.filter((script) => script.mediaType === 'application/ld+json');
+    const selectedScripts = scripts.slice(0, MAX_STRUCTURED_DATA_SCRIPTS);
+    const entities = new Map();
+    let charactersExamined = 0;
+    let documentsParsed = 0;
+    let malformedScripts = 0;
+    let externalScriptsSkipped = 0;
+    let objectsExamined = 0;
+    let arrayItemsExamined = 0;
+    let discardedProperties = 0;
+    let entityLimitReached = false;
+    let limitReached = scripts.length > MAX_STRUCTURED_DATA_SCRIPTS;
+    for (const script of selectedScripts) {
+        if (script.reference) {
+            externalScriptsSkipped += 1;
+            continue;
+        }
+        const remaining = MAX_STRUCTURED_DATA_TOTAL_CHARS - charactersExamined;
+        if (script.inlineContent.length === 0
+            || script.inlineContent.length > MAX_STRUCTURED_DATA_SCRIPT_CHARS
+            || script.inlineContent.length > remaining) {
+            limitReached = true;
+            continue;
+        }
+        charactersExamined += script.inlineContent.length;
+        if (!jsonStructureWithinBounds(script.inlineContent)) {
+            malformedScripts += 1;
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(script.inlineContent);
+            documentsParsed += 1;
+        }
+        catch {
+            malformedScripts += 1;
+            continue;
+        }
+        const pending = [{ value: parsed, depth: 0 }];
+        while (pending.length) {
+            const current = pending.pop();
+            if (!current)
+                break;
+            if (current.depth > MAX_STRUCTURED_DATA_DEPTH) {
+                limitReached = true;
+                continue;
+            }
+            if (Array.isArray(current.value)) {
+                if (arrayItemsExamined >= MAX_STRUCTURED_DATA_ARRAY_ITEMS) {
+                    limitReached = true;
+                    continue;
+                }
+                const remainingItems = MAX_STRUCTURED_DATA_ARRAY_ITEMS - arrayItemsExamined;
+                const retained = current.value.slice(0, remainingItems);
+                arrayItemsExamined += retained.length;
+                if (retained.length < current.value.length)
+                    limitReached = true;
+                for (const child of retained)
+                    pending.push({ value: child, depth: current.depth + 1 });
+                continue;
+            }
+            const object = record(current.value);
+            if (!object)
+                continue;
+            if (objectsExamined >= MAX_STRUCTURED_DATA_OBJECTS) {
+                limitReached = true;
+                continue;
+            }
+            objectsExamined += 1;
+            const entity = entityFromObject(object, baseUrl);
+            if (entity) {
+                const key = JSON.stringify(entity);
+                if (!entities.has(key)) {
+                    if (entities.size < MAX_STRUCTURED_DATA_ENTITIES)
+                        entities.set(key, entity);
+                    else {
+                        entityLimitReached = true;
+                        limitReached = true;
+                    }
+                }
+            }
+            const entries = Object.entries(object);
+            const traversed = entries.slice(0, MAX_STRUCTURED_DATA_PROPERTIES);
+            if (traversed.length < entries.length) {
+                discardedProperties += entries.length - traversed.length;
+                limitReached = true;
+            }
+            for (const [, child] of traversed) {
+                if (child !== null && typeof child === 'object') {
+                    pending.push({ value: child, depth: current.depth + 1 });
+                }
+            }
+        }
+    }
+    const truncated = input.sourceTruncated === true
+        || htmlAnalysis.inputLimitReached
+        || htmlAnalysis.scriptLimitReached
+        || htmlAnalysis.inlineLimitReached
+        || limitReached;
+    const partial = truncated || malformedScripts > 0 || externalScriptsSkipped > 0;
+    const limitations = [
+        'Structured identity fields are publisher-declared metadata and do not prove identity, ownership, control, safety, or maliciousness.',
+        'Only curated schema types, bounded labels, declared origins, and sameAs hostnames are retained; raw JSON-LD and arbitrary properties are discarded.',
+        'Static response evidence cannot observe JSON-LD added later by JavaScript.',
+    ];
+    if (input.sourceTruncated === true || htmlAnalysis.inputLimitReached) {
+        limitations.push('The captured homepage body was truncated, so structured identity evidence may be incomplete.');
+    }
+    if (htmlAnalysis.scriptLimitReached || scripts.length > MAX_STRUCTURED_DATA_SCRIPTS) {
+        limitations.push(`Only the first ${MAX_STRUCTURED_DATA_SCRIPTS} JSON-LD script elements were eligible for analysis.`);
+    }
+    if (htmlAnalysis.inlineLimitReached || limitReached) {
+        limitations.push(`JSON-LD analysis was bounded to ${MAX_STRUCTURED_DATA_SCRIPT_CHARS} characters per script and ${MAX_STRUCTURED_DATA_TOTAL_CHARS} characters in total.`);
+    }
+    if (malformedScripts > 0)
+        limitations.push(`${malformedScripts} JSON-LD script${malformedScripts === 1 ? ' was' : 's were'} malformed or exceeded the nesting boundary.`);
+    if (externalScriptsSkipped > 0)
+        limitations.push('Referenced JSON-LD scripts were not fetched.');
+    if (entityLimitReached) {
+        limitations.push(`Only the first ${MAX_STRUCTURED_DATA_ENTITIES} structured identity entities were retained.`);
+    }
+    return {
+        structuredDataVersion: STRUCTURED_DATA_IDENTITY_VERSION,
+        ...createObservation({
+            status: partial ? 'partial' : 'success',
+            observedAt: input.observedAt,
+            scanMode: 'deep',
+            source: 'html',
+            complete: !partial,
+            truncated,
+            limitations,
+            diagnostics: {
+                scriptsObserved: scripts.length,
+                scriptsExamined: selectedScripts.length,
+                charactersExamined,
+                documentsParsed,
+                malformedScripts,
+                externalScriptsSkipped,
+                objectsExamined,
+                arrayItemsExamined,
+                discardedProperties,
+                entities: entities.size,
+            },
+        }),
+        entities: [...entities.values()],
+    };
+}
+export { MAX_STRUCTURED_DATA_ARRAY_ITEMS, MAX_STRUCTURED_DATA_DEPTH, MAX_STRUCTURED_DATA_ENTITIES, MAX_STRUCTURED_DATA_OBJECTS, MAX_STRUCTURED_DATA_SAME_AS_HOSTS, MAX_STRUCTURED_DATA_SCRIPT_CHARS, MAX_STRUCTURED_DATA_SCRIPTS, MAX_STRUCTURED_DATA_TOTAL_CHARS, STRUCTURED_DATA_IDENTITY_VERSION, analyzeStructuredDataIdentity, };

package/lib/threat-intelligence-contract.mjs

@@ -0,0 +1,5 @@
+// Stable public facade for provider manifests, normalized results, matrices,
+// and the offline connector fixture harness.
+export * from "./threat-intelligence-runtime.mjs";
+export * from "./threat-intelligence-manifests.mjs";
+export * from "./threat-intelligence-fixtures.mjs";

package/lib/threat-intelligence-definition-registry.mjs

@@ -0,0 +1,34 @@
+const PROVIDER_DEFINITIONS = new WeakSet();
+const CURATED_CONNECTOR_DEFINITIONS = new WeakSet();
+export function registerThreatIntelligenceProvider(definition) {
+    PROVIDER_DEFINITIONS.add(definition);
+    return definition;
+}
+export function assertThreatIntelligenceProvider(provider) {
+    if (!provider ||
+        typeof provider !== 'object' ||
+        !PROVIDER_DEFINITIONS.has(provider)) {
+        throw new TypeError('A versioned threat-intelligence provider definition is required');
+    }
+}
+export function isThreatIntelligenceProvider(provider) {
+    return Boolean(provider &&
+        typeof provider === 'object' &&
+        PROVIDER_DEFINITIONS.has(provider));
+}
+export function registerCuratedConnector(definition) {
+    CURATED_CONNECTOR_DEFINITIONS.add(definition);
+    return definition;
+}
+export function assertCuratedConnectorDefinition(connector) {
+    if (!connector ||
+        typeof connector !== 'object' ||
+        !CURATED_CONNECTOR_DEFINITIONS.has(connector)) {
+        throw new TypeError('A versioned curated connector definition is required');
+    }
+}
+export function isCuratedConnectorDefinition(connector) {
+    return Boolean(connector &&
+        typeof connector === 'object' &&
+        CURATED_CONNECTOR_DEFINITIONS.has(connector));
+}

package/lib/threat-intelligence-fixtures.mjs

@@ -0,0 +1,82 @@
+import { createCuratedConnectorResult, normalizeCuratedConnectorTarget, } from "./threat-intelligence-runtime.mjs";
+import { assertCuratedConnectorDefinition } from "./threat-intelligence-definition-registry.mjs";
+const MAX_CONNECTOR_KEY_LENGTH = 80;
+const MAX_CONNECTOR_FIXTURE_BYTES = 512 * 1024;
+function isRecord(value) {
+    return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+function exactFixtureKeys(value) {
+    if (!isRecord(value))
+        throw new TypeError('Connector fixture must be an object');
+    const allowed = new Set(['id', 'target', 'observedAt', 'json']);
+    for (const key of Object.keys(value)) {
+        if (!allowed.has(key)) {
+            throw new TypeError(`Connector fixture has unknown field: ${key}`);
+        }
+    }
+}
+function fixtureId(value) {
+    if (typeof value !== 'string' ||
+        value.length > MAX_CONNECTOR_KEY_LENGTH ||
+        !/^[a-z0-9][a-z0-9._-]*$/iu.test(value)) {
+        return null;
+    }
+    return value;
+}
+function fixtureTimestamp(value) {
+    if (typeof value !== 'string' || value.length > 50)
+        return null;
+    const timestamp = Date.parse(value);
+    return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null;
+}
+export function runCuratedConnectorFixture(connector, fixture, normalizeFixture) {
+    assertCuratedConnectorDefinition(connector);
+    exactFixtureKeys(fixture);
+    const id = fixtureId(fixture.id);
+    const observedAt = fixtureTimestamp(fixture.observedAt);
+    if (!id || !observedAt) {
+        throw new TypeError('Connector fixture identity or timestamp is invalid');
+    }
+    if (typeof fixture.json !== 'string' ||
+        Buffer.byteLength(fixture.json, 'utf8') > MAX_CONNECTOR_FIXTURE_BYTES) {
+        throw new TypeError(`Connector fixture JSON must not exceed ${MAX_CONNECTOR_FIXTURE_BYTES} bytes`);
+    }
+    if (typeof normalizeFixture !== 'function') {
+        throw new TypeError('Connector fixture requires a synchronous normalizer');
+    }
+    let firstPayload;
+    let secondPayload;
+    try {
+        firstPayload = JSON.parse(fixture.json);
+        secondPayload = JSON.parse(fixture.json);
+    }
+    catch {
+        throw new TypeError('Connector fixture JSON is invalid');
+    }
+    const targetType = isRecord(fixture.target) && typeof fixture.target.type === 'string'
+        ? fixture.target.type
+        : null;
+    const declaration = targetType
+        ? connector.inputs.find((item) => item.type === targetType)
+        : null;
+    if (!declaration) {
+        throw new TypeError('Connector fixture target is unsupported');
+    }
+    const normalizedTarget = normalizeCuratedConnectorTarget(fixture.target, declaration.exposure);
+    const normalizeOnce = (payload) => {
+        const output = normalizeFixture(payload, Object.freeze({ ...normalizedTarget }));
+        if (output &&
+            (typeof output === 'object' || typeof output === 'function') &&
+            typeof output.then === 'function') {
+            throw new TypeError('Connector fixture normalizers must be synchronous');
+        }
+        return createCuratedConnectorResult(connector, fixture.target, output, observedAt);
+    };
+    const first = normalizeOnce(firstPayload);
+    const second = normalizeOnce(secondPayload);
+    if (JSON.stringify(first) !== JSON.stringify(second)) {
+        throw new TypeError(`Connector fixture ${id} is not deterministic`);
+    }
+    return first;
+}
+export { MAX_CONNECTOR_FIXTURE_BYTES };