@slicedearth/whoisleuth-cli 2.2.0 → 2.4.0

package/cli/lookup-command-runner.mjs

@@ -2,7 +2,7 @@ import { abortable } from "../lib/abort.mjs";
 import { classifyQuery } from "../lib/classify.mjs";
 import { runUnifiedLookup } from "../lib/lookup.mjs";
 import { plannedLookupProgressSources } from "../lib/lookup-source-progress.mjs";
-import { CliUsageError, boundedCliErrorMessage } from "./errors.mjs";
+import { CliUsageError } from "./errors.mjs";
 import EXIT_CODES from "./exit-codes.mjs";
 import { buildCliEvidenceExport } from "./export-evidence.mjs";
 import { formatLookupEvidenceHtml } from "./formatters/html.mjs";
@@ -17,6 +17,7 @@ import { evaluateCliFailPolicies, formatFailPolicyNotice } from "./fail-policy.m
 import { formatCliJunit } from "./ci-report.mjs";
 import { browseLookupOperation, canBrowseLookup } from "./lookup-browser.mjs";
 import { writePrivateFile } from "./output-file.mjs";
+import { prepareLookupCollectionTarget, prepareSelectedLookupUrl } from "../packages/evidence/lookup-target.mjs";
 async function runLookupCommand(args, dependencies, context) {
     const browserInput = dependencies.stdin || process.stdin;
     const browserEnvironment = dependencies.environment || process.env;
@@ -35,23 +36,31 @@ async function runLookupCommand(args, dependencies, context) {
     });
     context.setEventProgress(eventProgress);
     eventProgress.emit({ event: 'started' });
-    const query = args.query || await context.readSingleInput();
-    if (!query)
+    const input = args.query || await context.readSingleInput();
+    if (!input)
         throw new CliUsageError('lookup requires one domain, IP address, or ASN as an argument or on stdin.');
     const classify = dependencies.classifyQuery || classifyQuery;
     const executeLookup = dependencies.runUnifiedLookup || runUnifiedLookup;
     let classified;
+    let query;
+    let selectedUrl;
     try {
+        query = prepareLookupCollectionTarget(input);
         classified = classify(query);
+        if (args.exactUrl) {
+            if (!args.deep || classified.type !== 'domain')
+                throw new TypeError();
+            selectedUrl = prepareSelectedLookupUrl(input, classified.inputHostname);
+        }
     }
-    catch (error) {
-        throw new CliUsageError(boundedCliErrorMessage(error, 'Invalid query'));
+    catch {
+        throw new CliUsageError(args.exactUrl ? '--exact-url requires --deep and an HTTP(S) domain URL without credentials or a custom port.' : 'Invalid domain, IP address, ASN or credential-free HTTP(S) URL.');
     }
     if ((args.output === 'markdown' || args.output === 'html') && classified.type !== 'domain') {
         throw new CliUsageError('Markdown and HTML reports support domain lookups only.');
     }
     if (args.plan) {
-        const plan = buildCliLookupPlan(query, classified, args.deep);
+        const plan = buildCliLookupPlan(query, classified, args.deep, Boolean(selectedUrl));
         context.writeStdout(args.output === 'json'
             ? formatJsonDocument(plan)
             : context.terminal(formatCliLookupPlan(plan), args.color));
@@ -83,6 +92,7 @@ async function runLookupCommand(args, dependencies, context) {
                     ? {
                         fast: false,
                         compact: false,
+                        ...(selectedUrl ? { selectedUrl } : {}),
                         signal,
                         onSourceSettled: (settlement) => {
                             onSourceSettled(settlement);
@@ -119,6 +129,7 @@ async function runLookupCommand(args, dependencies, context) {
                 ? {
                     fast: false,
                     compact: false,
+                    ...(selectedUrl ? { selectedUrl } : {}),
                     ...(dependencies.signal ? { signal: dependencies.signal } : {}),
                     onSourceSettled: (settlement) => {
                         settledSources += 1;

package/cli/lookup-diff.mjs

@@ -4,6 +4,7 @@ import { buildBulkDomainComparison, } from "../packages/investigation/bulk-domai
 import { relationshipObservation } from "../packages/comparison/relationship-evidence.mjs";
 import { CliUsageError } from "./errors.mjs";
 import { parseSavedLookupDocument } from "./saved-lookup.mjs";
+import { lookupObservationHostname } from "../packages/evidence/lookup-target.mjs";
 export const CLI_LOOKUP_DIFF_SCHEMA = 'whoisleuth.cli.lookup-diff';
 export const CLI_LOOKUP_DIFF_VERSION = 1;
 function text(value, maximum = 300) {
@@ -129,19 +130,22 @@ function lookupComparisonInput(document) {
 function buildCliLookupDiff(leftText, rightText, generatedAt = new Date().toISOString(), options = {}) {
     const left = parseSavedLookupDocument(leftText, { label: 'Left lookup input' });
     const right = parseSavedLookupDocument(rightText, { label: 'Right lookup input' });
-    const domainMode = options.domainMode ?? 'different';
+    const domainMode = options.domainMode === 'auto'
+        ? left.registrableDomain === right.registrableDomain ? 'same' : 'different'
+        : options.domainMode ?? 'different';
     if (domainMode === 'different' && left.registrableDomain === right.registrableDomain) {
         throw new CliUsageError('Lookup diff requires documents for two different domains.');
     }
     if (domainMode === 'same' && left.registrableDomain !== right.registrableDomain) {
         throw new CliUsageError('Lookup history requires observations for the same domain.');
     }
-    const comparison = buildBulkDomainComparison(lookupComparisonInput(left), lookupComparisonInput(right), domainMode === 'same' ? right.generatedAt : null, {
+    const measuredComparison = buildBulkDomainComparison(lookupComparisonInput(left), lookupComparisonInput(right), domainMode === 'same' ? right.generatedAt : null, {
         allowSameDomain: domainMode === 'same',
         ...(domainMode === 'same' ? { now: Date.parse(right.generatedAt) } : {}),
     });
-    if (!comparison)
+    if (!measuredComparison)
         throw new CliUsageError('Lookup documents could not be compared.');
+    const comparison = qualifyLookupComparison(measuredComparison, left, right, domainMode === 'same');
     return {
         schema: CLI_LOOKUP_DIFF_SCHEMA,
         version: CLI_LOOKUP_DIFF_VERSION,
@@ -160,6 +164,36 @@ function buildCliLookupDiff(leftText, rightText, generatedAt = new Date().toISOS
         ],
     };
 }
+function qualifyLookupComparison(comparison, left, right, temporal) {
+    const leftScope = recordOrEmpty(left.availability);
+    const rightScope = recordOrEmpty(right.availability);
+    const hostnameChanged = temporal && lookupObservationHostname(leftScope) !== lookupObservationHostname(rightScope);
+    const selectedPage = leftScope.webObservationMode === 'selected_url' || rightScope.webObservationMode === 'selected_url';
+    if (!hostnameChanged && !selectedPage)
+        return comparison;
+    const limitation = 'The retained collection hostname differs or a selected URL has no retained complete path/query identity. Values remain visible, but those fields cannot establish change at the same target.';
+    const rows = comparison.rows.map((row) => {
+        const web = ['web', 'technology', 'identity'].includes(row.category);
+        const incomparable = hostnameChanged && !['registration', 'lifecycle', 'source'].includes(row.category)
+            || temporal && selectedPage && web;
+        return {
+            ...row,
+            ...(selectedPage && web ? { method: 'Bounded website observation; one or both captures concern a selected URL rather than the homepage' } : {}),
+            ...(incomparable ? {
+                state: 'not_recorded',
+                source: `${row.source}; comparable target identity not recorded`,
+                leftSourceState: 'not_recorded',
+                rightSourceState: 'not_recorded',
+                limitations: [...row.limitations, limitation],
+            } : {}),
+        };
+    });
+    return {
+        ...comparison, rows,
+        counts: Object.fromEntries(Object.keys(comparison.counts).map((state) => [state, rows.filter((row) => row.state === state).length])),
+        limitations: [...comparison.limitations, limitation],
+    };
+}
 function formatCliLookupDiff(document) {
     const rows = document.comparison.rows.filter((row) => row.state !== 'equal');
     const output = [

package/cli/lookup-plan.mjs

@@ -21,7 +21,7 @@ const FAST_DOMAIN_EVIDENCE = Object.freeze({
 const DEEP_DOMAIN_EVIDENCE = Object.freeze({
     source: 'domain_evidence',
     purpose: 'Collect bounded DNS, HTTP, TLS, page-identity, technology, and security-posture evidence.',
-    disclosure: 'DNS resolvers and the target website infrastructure receive the hostname through bounded probes.',
+    disclosure: 'DNS, TLS and website probes use the submitted hostname. Registration-delegation queries use the registrable domain.',
     conditional: false,
 });
 const REGISTRAR_RDAP = Object.freeze({
@@ -42,19 +42,23 @@ const REVERSE_DNS = Object.freeze({
     disclosure: 'A DNS resolver receives the reverse lookup for the normalised IP address.',
     conditional: false,
 });
-function plannedSources(classified, deep) {
+function plannedSources(classified, deep, selectedUrl) {
     if (!deep) {
         return Object.freeze(classified.type === 'domain' ? [RDAP, FAST_DOMAIN_EVIDENCE] : [RDAP]);
     }
     if (classified.type === 'domain') {
-        return Object.freeze([RDAP, WHOIS, DEEP_DOMAIN_EVIDENCE, REGISTRAR_RDAP, NETWORK_CONTEXT]);
+        const domainEvidence = selectedUrl ? Object.freeze({
+            ...DEEP_DOMAIN_EVIDENCE,
+            disclosure: 'The selected URL path and query are sent to the website and followed redirects. The fragment is not sent. DNS and TLS use its hostname; registration uses the registrable domain. Saved HTTP provenance omits queries, but page-derived text may contain sensitive information.',
+        }) : DEEP_DOMAIN_EVIDENCE;
+        return Object.freeze([RDAP, WHOIS, domainEvidence, REGISTRAR_RDAP, NETWORK_CONTEXT]);
     }
     if (classified.type === 'ipv4' || classified.type === 'ipv6') {
         return Object.freeze([RDAP, WHOIS, REVERSE_DNS]);
     }
     return Object.freeze([RDAP, WHOIS]);
 }
-function buildCliLookupPlan(query, classified, deep) {
+function buildCliLookupPlan(query, classified, deep, selectedUrl = false) {
     return Object.freeze({
         schema: CLI_LOOKUP_PLAN_SCHEMA,
         version: CLI_LOOKUP_PLAN_VERSION,
@@ -69,7 +73,7 @@ function buildCliLookupPlan(query, classified, deep) {
         planning: Object.freeze({
             networkRequestsMade: false,
             collectionRequiresNetwork: true,
-            sources: plannedSources(classified, deep),
+            sources: plannedSources(classified, deep, selectedUrl),
         }),
         limitations: Object.freeze([
             'This is a local preflight. It does not test source availability, feature configuration, cache state, redirects, referrals, or the exact number of requests a completed lookup may require.',
@@ -81,6 +85,8 @@ function formatCliLookupPlan(plan) {
     const lines = [
         'WHOISleuth lookup preflight',
         `Target: ${plan.target.normalized}`,
+        ...(plan.target.inputHostname && plan.target.inputHostname !== plan.target.normalized
+            ? [`Submitted hostname: ${plan.target.inputHostname}`] : []),
         `Type: ${plan.target.type}`,
         `Mode: ${plan.mode}`,
         'Network requests made: no',

package/cli/lookup-reconcile.mjs

@@ -1,6 +1,7 @@
 import { CliUsageError } from "./errors.mjs";
 import { buildCliLookupDiff } from "./lookup-diff.mjs";
 import { parseSavedLookupDocument } from "./saved-lookup.mjs";
+import { compareObservationContexts } from "../packages/comparison/capture-context.mjs";
 const CLI_LOOKUP_RECONCILIATION_SCHEMA = 'whoisleuth.cli.lookup-reconciliation';
 const CLI_LOOKUP_RECONCILIATION_VERSION = 1;
 const MAX_LOOKUP_RECONCILIATION_INPUT_BYTES = 32 * 1024 * 1024;
@@ -79,15 +80,11 @@ function buildCliLookupReconciliation(inputs, generatedAt = new Date().toISOStri
         };
     });
     const observations = documents.map(observationContext);
-    const identities = observations.map((observation) => (observation.observerLabel && observation.vantageLabel
-        ? `${observation.observerLabel}\u0000${observation.vantageLabel}`
-        : null));
-    const labelsComplete = identities.every((identity) => identity !== null);
-    const labelsDistinct = labelsComplete && new Set(identities).size === identities.length;
-    const independence = labelsDistinct
+    const context = compareObservationContexts(observations.map(observation => ({ ...observation, observedAt: observation.generatedAt })));
+    const independence = context.labels === 'distinct'
         ? {
             state: 'verified_distinct_labels',
-            reason: 'Every observation carries a different analyst-supplied observer and vantage label.',
+            reason: 'Every observation carries a distinct analyst-supplied observer/vantage label pair. Actual collection independence is not verified.',
         }
         : {
             state: 'unverified',
@@ -117,15 +114,19 @@ function buildCliLookupReconciliation(inputs, generatedAt = new Date().toISOStri
     };
 }
 function formatCliLookupReconciliation(document) {
+    const context = compareObservationContexts(document.observations.map(observation => ({ ...observation, observedAt: observation.generatedAt })));
     const output = [
         'Lookup observation reconciliation',
         `Domain             ${document.domain}`,
         `Observations       ${document.summary.observationCount}`,
-        `Independence       ${document.independence.state.replaceAll('_', ' ')}`,
+        `Declared labels    ${context.labels}; collection independence not verified`,
+        `Capture time span  ${context.time.spanMilliseconds === null ? 'unknown' : `${context.time.spanMilliseconds / 1000} seconds`}`,
         `Agreement          ${document.summary.agreement}`,
         `Disagreement       ${document.summary.disagreement}`,
         `Non-comparable     ${document.summary.nonComparable}`,
         '',
+        ...document.observations.map((observation, index) => `Observation ${index + 1}: ${observation.generatedAt} · ${observation.mode} · observer ${observation.observerLabel ?? 'not declared'} · vantage ${observation.vantageLabel ?? 'not declared'}`),
+        '',
     ];
     for (const field of document.fields.filter((candidate) => candidate.state !== 'agreement')) {
         output.push(`${field.label} [${field.state.replaceAll('_', ' ')}]`);
@@ -136,6 +137,7 @@ function formatCliLookupReconciliation(document) {
     output.push('', 'Limitations:');
     for (const limitation of document.limitations)
         output.push(`  - ${limitation}`);
+    output.push('  - These selected observations do not establish worldwide availability, removal or takedown.');
     return `${output.join('\n')}\n`;
 }
 export { CLI_LOOKUP_RECONCILIATION_SCHEMA, CLI_LOOKUP_RECONCILIATION_VERSION, MAX_LOOKUP_RECONCILIATION_INPUT_BYTES, buildCliLookupReconciliation, formatCliLookupReconciliation, };