@slicedearth/whoisleuth-cli 2.4.0 → 2.6.0

package/cli/collection-command-definitions.mjs

@@ -0,0 +1,330 @@
+import { positional, OPTIONAL_FILE_POSITIONAL, OPTIONAL_TEXT_POSITIONAL, constraint, commandSeed, } from "./command-definition.mjs";
+import { bulkQueryLimit, bulkConcurrencyLimit } from "../lib/bulk-limits.mjs";
+const COLLECTION_CEILINGS = `${bulkQueryLimit('fast')} fast or ${bulkQueryLimit('deep')} deep`;
+const CONCURRENCY_CEILINGS = `${bulkConcurrencyLimit('fast')} fast or ${bulkConcurrencyLimit('deep')} deep`;
+export const COLLECTION_COMMAND_DEFINITIONS = Object.freeze({
+    lookup: commandSeed({
+        reference: {
+            description: 'Collect registration evidence for one domain, IP, or ASN.',
+            example: 'whoisleuth lookup example.test --deep --browse',
+            boundary: 'Fast is the default. An ICANN-recognised public domain, reserved documentation domain, IP, or ASN may occupy command position as shorthand; it delegates to this same parser and URL-like input requires the explicit lookup command. Deep mode adds bounded WHOIS, DNS, HTTP, TLS, technology, posture, and network context where applicable. --deep --exact-url explicitly sends the input URL path and query to the website, without its fragment; ordinary URL input sends only the hostname. --plan remains offline and omits the selected URL. Retained paths and page-derived text require review before sharing. A full Deep homepage observation can derive fixed publication and delivery/cache summaries from the same response without retaining raw metadata values or making another request. --browse opens before collection, shows aggregate Fast progress or independently settled planned Deep sources, and then navigates allowlisted retained fields in the completed document. Press ? for help and / to search rendered panel text only. Closing during collection cancels without a partial document. --save-lookup writes the exact completed private JSON only after a normal browser close; it can contain normalised evidence omitted from panels and refuses an existing path.',
+        },
+        collection: {
+            scope: 'Accepts one target. Fast is the default; deep collection must be selected explicitly.',
+        },
+        summary: 'Collect one domain, IP, or ASN',
+        options: [
+            '--json',
+            '--junit',
+            '--markdown',
+            '--html',
+            '--no-attribution',
+            '--fast',
+            '--deep',
+            '--exact-url',
+            '--observer',
+            '--vantage',
+            '--plan',
+            '--summary',
+            '--verbose',
+            '--browse',
+            '--save-lookup',
+            '--strict-exit',
+            '--fail-on',
+            '--events',
+            '--quiet',
+            '--no-color',
+        ],
+        positionals: Object.freeze([
+            positional('target', 'text', 0, 1, [], 'argv_or_stdin', ['--browse']),
+        ]),
+        constraints: Object.freeze([
+            constraint({
+                kind: 'mutually_exclusive',
+                options: ['--json', '--junit', '--markdown', '--html'],
+            }),
+            constraint({ kind: 'mutually_exclusive', options: ['--fast', '--deep'] }),
+            constraint({ kind: 'mutually_exclusive', options: ['--summary', '--verbose'] }),
+            constraint({ kind: 'requires_all', option: '--save-lookup', requiredOptions: ['--browse'] }),
+            constraint({ kind: 'requires_all', option: '--exact-url', requiredOptions: ['--deep'] }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--summary',
+                excludedOptions: ['--json', '--junit', '--markdown', '--html'],
+            }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--verbose',
+                excludedOptions: ['--json', '--junit', '--markdown', '--html'],
+            }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--browse',
+                excludedOptions: [
+                    '--json',
+                    '--junit',
+                    '--markdown',
+                    '--html',
+                    '--summary',
+                    '--verbose',
+                    '--events',
+                    '--plan',
+                    '--quiet',
+                ],
+            }),
+            constraint({
+                kind: 'requires_any',
+                option: '--no-attribution',
+                requiredOptions: ['--markdown', '--html'],
+            }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--plan',
+                excludedOptions: [
+                    '--junit',
+                    '--markdown',
+                    '--html',
+                    '--summary',
+                    '--verbose',
+                    '--strict-exit',
+                    '--events',
+                    '--quiet',
+                    '--fail-on',
+                ],
+            }),
+        ]),
+        handlerOwner: 'lookup',
+        networkEffect: 'conditional_network',
+        common: true,
+        schemaIdentifiers: Object.freeze(['whoisleuth.cli.lookup', 'whoisleuth.cli.lookup-plan']),
+        primaryArtefacts: Object.freeze(['Source-qualified Lookup', 'Lookup request plan']),
+        planSupport: true,
+        additionalOutputFormats: Object.freeze([]),
+        bootstrapProfile: 'allowed',
+    }),
+    bulk: commandSeed({
+        reference: {
+            description: 'Triage newline-delimited domains, IPs, or ASNs with bounded concurrency.',
+            example: 'cat domains.txt | whoisleuth bulk --jsonl',
+            boundary: 'Fast and deep jobs use separate concurrency ceilings. Filters affect output only; collection failures and inconclusive authority states remain explicit in JSON, JSONL, and CSV. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns.',
+        },
+        collection: {
+            scope: `Accepts at most ${COLLECTION_CEILINGS} targets, with concurrency capped at ${CONCURRENCY_CEILINGS}.`,
+        },
+        summary: 'Run bounded multi-target collection',
+        options: [
+            '--json',
+            '--jsonl',
+            '--junit',
+            '--csv',
+            '--csv-with-metadata',
+            '--domains',
+            '--queries',
+            '--registered-only',
+            '--inconclusive-only',
+            '--errors-only',
+            '--fast',
+            '--deep',
+            '--concurrency',
+            '--checkpoint',
+            '--resume',
+            '--events',
+            '--plan',
+            '--fail-on',
+            '--quiet',
+            '--no-color',
+        ],
+        positionals: OPTIONAL_FILE_POSITIONAL,
+        constraints: Object.freeze([
+            constraint({
+                kind: 'mutually_exclusive',
+                options: [
+                    '--json',
+                    '--jsonl',
+                    '--junit',
+                    '--csv',
+                    '--csv-with-metadata',
+                    '--domains',
+                    '--queries',
+                ],
+            }),
+            constraint({
+                kind: 'mutually_exclusive',
+                options: ['--registered-only', '--inconclusive-only', '--errors-only'],
+            }),
+            constraint({ kind: 'mutually_exclusive', options: ['--fast', '--deep'] }),
+            constraint({ kind: 'requires_all', option: '--resume', requiredOptions: ['--checkpoint'] }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--plan',
+                excludedOptions: [
+                    '--jsonl',
+                    '--junit',
+                    '--csv',
+                    '--csv-with-metadata',
+                    '--domains',
+                    '--queries',
+                    '--events',
+                    '--checkpoint',
+                    '--resume',
+                    '--quiet',
+                    '--fail-on',
+                ],
+            }),
+        ]),
+        handlerOwner: 'bulk',
+        networkEffect: 'conditional_network',
+        common: true,
+        schemaIdentifiers: Object.freeze([
+            'whoisleuth.cli.bulk',
+            'whoisleuth.cli.bulk.item',
+            'whoisleuth.cli.bulk-checkpoint',
+        ]),
+        primaryArtefacts: Object.freeze(['Bulk result', 'Bulk checkpoint']),
+        planSupport: true,
+        additionalOutputFormats: Object.freeze([]),
+        bootstrapProfile: 'allowed',
+    }),
+    discover: commandSeed({
+        reference: {
+            description: 'Generate bounded lookalike-domain candidates from a brand or registrable domain, including multi-part public suffixes.',
+            example: 'whoisleuth discover example.test --preset common --jsonl',
+            boundary: 'Generation and optional local snapshot comparison are offline. Candidates are leads only and are not resolved, registered, or classified as malicious.',
+        },
+        collection: {
+            scope: 'Generates a bounded candidate set from local rules, dictionaries, and optional saved snapshots.',
+        },
+        summary: 'Generate lookalike candidates offline',
+        options: [
+            '--tlds',
+            '--preset',
+            '--families',
+            '--keyboard',
+            '--dictionary',
+            '--snapshot',
+            '--json',
+            '--jsonl',
+            '--domains',
+            '--quiet',
+            '--no-color',
+        ],
+        positionals: OPTIONAL_TEXT_POSITIONAL,
+        constraints: Object.freeze([
+            constraint({ kind: 'mutually_exclusive', options: ['--json', '--jsonl', '--domains'] }),
+            constraint({ kind: 'mutually_exclusive', options: ['--preset', '--families'] }),
+            constraint({
+                kind: 'value_excludes',
+                option: '--preset',
+                value: 'common',
+                excludedOptions: ['--dictionary'],
+            }),
+        ]),
+        handlerOwner: 'discovery',
+        networkEffect: 'offline',
+        common: true,
+        schemaIdentifiers: Object.freeze([
+            'whoisleuth.cli.discover',
+            'whoisleuth.cli.discover.item',
+            'whoisleuth.cli.discovery-snapshot',
+        ]),
+        primaryArtefacts: Object.freeze(['Candidate set', 'Discovery snapshot']),
+        planSupport: false,
+        additionalOutputFormats: Object.freeze([]),
+        bootstrapProfile: 'allowed',
+    }),
+    'discover-scan': commandSeed({
+        reference: {
+            description: 'Generate a bounded candidate set, collect a selected subset, and produce a supervised review queue.',
+            example: 'whoisleuth discover-scan example.test --scan-limit 50 --checkpoint scan.json --json',
+            boundary: `This command performs network collection. Fast compact lookup is the default; deep mode is capped at ${bulkQueryLimit('deep')} candidates. Allowlisting changes review priority only and shared infrastructure remains a lead, not attribution. --csv-with-metadata adds source versions, separate observation and report times, collection origin and diagnostic states; --csv retains the compact columns.`,
+        },
+        collection: {
+            scope: `Scans at most ${COLLECTION_CEILINGS} candidates, with concurrency capped at ${CONCURRENCY_CEILINGS}.`,
+        },
+        summary: 'Collect a supervised candidate review queue',
+        options: [
+            '--tlds',
+            '--preset',
+            '--families',
+            '--keyboard',
+            '--dictionary',
+            '--fast',
+            '--deep',
+            '--scan-limit',
+            '--chunk-size',
+            '--concurrency',
+            '--resolver',
+            '--allowlist',
+            '--checkpoint',
+            '--resume',
+            '--observation-snapshot',
+            '--registered-only',
+            '--inconclusive-only',
+            '--acquisition-only',
+            '--suppressed-only',
+            '--events',
+            '--plan',
+            '--fail-on',
+            '--json',
+            '--jsonl',
+            '--csv',
+            '--csv-with-metadata',
+            '--domains',
+            '--quiet',
+            '--no-color',
+        ],
+        positionals: OPTIONAL_TEXT_POSITIONAL,
+        constraints: Object.freeze([
+            constraint({
+                kind: 'mutually_exclusive',
+                options: ['--json', '--jsonl', '--csv', '--csv-with-metadata', '--domains'],
+            }),
+            constraint({ kind: 'mutually_exclusive', options: ['--preset', '--families'] }),
+            constraint({ kind: 'mutually_exclusive', options: ['--fast', '--deep'] }),
+            constraint({
+                kind: 'mutually_exclusive',
+                options: [
+                    '--registered-only',
+                    '--inconclusive-only',
+                    '--acquisition-only',
+                    '--suppressed-only',
+                ],
+            }),
+            constraint({ kind: 'requires_all', option: '--resume', requiredOptions: ['--checkpoint'] }),
+            constraint({
+                kind: 'value_excludes',
+                option: '--preset',
+                value: 'common',
+                excludedOptions: ['--dictionary'],
+            }),
+            constraint({
+                kind: 'excludes_all',
+                option: '--plan',
+                excludedOptions: [
+                    '--jsonl',
+                    '--csv',
+                    '--csv-with-metadata',
+                    '--domains',
+                    '--events',
+                    '--checkpoint',
+                    '--resume',
+                    '--observation-snapshot',
+                    '--quiet',
+                    '--fail-on',
+                ],
+            }),
+        ]),
+        handlerOwner: 'discovery_scan',
+        networkEffect: 'conditional_network',
+        common: true,
+        schemaIdentifiers: Object.freeze([
+            'whoisleuth.cli.discovery-scan',
+            'whoisleuth.cli.discovery-scan.item',
+            'whoisleuth.cli.discovery-observation-snapshot',
+        ]),
+        primaryArtefacts: Object.freeze(['Reviewed candidate queue', 'Observation snapshot']),
+        planSupport: true,
+        additionalOutputFormats: Object.freeze([]),
+        bootstrapProfile: 'allowed',
+    }),
+});

package/cli/collection-preflight.mjs

@@ -1,3 +1,4 @@
+import { bulkQueryLimit } from "../lib/bulk-limits.mjs";
 export const CLI_COLLECTION_PREFLIGHT_SCHEMA = 'whoisleuth.cli.collection-preflight';
 export const CLI_COLLECTION_PREFLIGHT_VERSION = 1;
 export function buildCollectionPreflight(options) {
@@ -12,7 +13,7 @@ export function buildCollectionPreflight(options) {
         networkRequestsMade: false,
         scope: Object.freeze({
             selectedTargets: options.targetCount,
-            commandTargetLimit: options.targetLimit,
+            commandTargetLimit: bulkQueryLimit(options.deep ? 'deep' : 'fast'),
             concurrency: options.concurrency,
             sourceFamilies: Object.freeze(sourceFamilies),
             exactRequestCountKnown: false,
@@ -53,6 +54,12 @@ export function formatCollectionPreflight(document) {
         ...document.scope.sourceFamilies.map((source) => `  ${source}`),
         '',
         ...document.disclosure.map((item) => `Disclosure: ${item}`),
+        '',
+        `Output format: ${document.persistence.output}`,
+        `Checkpoint: ${document.persistence.checkpoint ? 'enabled' : 'not written'}. ${document.persistence.note}`,
+        ...(document.review.allowlistAffectsPriorityOnly ? ['Allowlist: affects review priority only; it does not exclude targets from collection.'] : []),
+        'Enforcement: none. This plan does not report, block or remove a target.',
+        '',
         ...document.limitations.map((item) => `Limitation: ${item}`),
         '',
     ].join('\n');

package/cli/command-argument-grammar.mjs

@@ -1,215 +1,13 @@
 import { CliUsageError } from "./errors.mjs";
-import { commandDefinition, } from "./command-reference.mjs";
-function humanList(values, conjunction = 'or') {
-    if (values.length < 2)
-        return values[0] ?? '';
-    if (values.length === 2)
-        return `${values[0]} ${conjunction} ${values[1]}`;
-    return `${values.slice(0, -1).join(', ')}, ${conjunction} ${values.at(-1)}`;
-}
-function valueError(specification) {
-    if (specification.valueKind === 'file') {
-        return new CliUsageError(`${specification.option} requires one bounded file path.`);
-    }
-    if (specification.valueKind === 'integer') {
-        const baseRange = specification.integerRanges[0];
-        return new CliUsageError(baseRange
-            ? `${specification.option} requires an integer from ${baseRange.minimum} to ${baseRange.maximum}.`
-            : `${specification.option} requires one bounded integer.`);
-    }
-    if (specification.values.length > 0) {
-        return new CliUsageError(`${specification.option} requires ${humanList(specification.values)}.`);
-    }
-    return new CliUsageError(`${specification.option} requires one bounded value.`);
-}
-function parseTokens(command, argv) {
-    const specificationByOption = new Map(commandDefinition(command).grammar.options.map((specification) => [specification.option, specification]));
-    const options = new Map();
-    const positionals = [];
-    for (let index = 0; index < argv.length; index += 1) {
-        const argument = argv[index];
-        if (argument === '--') {
-            positionals.push(...argv.slice(index + 1));
-            break;
-        }
-        if (!argument.startsWith('-')) {
-            positionals.push(argument);
-            continue;
-        }
-        const specification = specificationByOption.get(argument);
-        if (!specification || specification.metaAction !== null) {
-            throw new CliUsageError(`Unknown option "${argument}".`);
-        }
-        const existing = options.get(argument);
-        if (existing && specification.occurrence === 'once') {
-            throw new CliUsageError(`${argument} may be supplied only once.`);
-        }
-        if (existing && specification.occurrence === 'idempotent')
-            continue;
-        let value = '';
-        if (specification.arity === 1) {
-            const candidate = argv[++index];
-            if (candidate === undefined || candidate === ''
-                || (!specification.acceptsOptionLikeValue && candidate.startsWith('-'))) {
-                throw valueError(specification);
-            }
-            value = candidate;
-        }
-        if (existing)
-            existing.push(value);
-        else
-            options.set(argument, [value]);
-    }
-    return { options, positionals };
-}
-function selectedIntegerRange(specification, options) {
-    return specification.integerRanges
-        .filter((range) => range.whenOptionPresent === null || options.has(range.whenOptionPresent))
-        .at(-1) ?? null;
-}
-function validateOptionValues(command, options) {
-    for (const specification of commandDefinition(command).grammar.options) {
-        const values = options.get(specification.option);
-        if (!values || specification.arity === 0)
-            continue;
-        if (specification.valueKind === 'enum') {
-            if (values.some((value) => !specification.values.includes(value)))
-                throw valueError(specification);
-            continue;
-        }
-        if (specification.valueKind === 'integer') {
-            const range = selectedIntegerRange(specification, options);
-            if (!range)
-                throw new Error(`Integer option ${command} ${specification.option} has no applicable range.`);
-            for (const value of values) {
-                if (!/^\d+$/u.test(value))
-                    throw valueError(specification);
-                const parsed = Number(value);
-                if (!Number.isSafeInteger(parsed) || parsed < range.minimum || parsed > range.maximum) {
-                    throw new CliUsageError(`${specification.option} must be from ${range.minimum} to ${range.maximum}.`);
-                }
-            }
-            continue;
-        }
-        if (specification.valueKind === 'policy_list') {
-            for (const value of values) {
-                const policies = [...new Set(value.split(',').map((item) => item.trim().toLowerCase()).filter(Boolean))];
-                if (!policies.length || policies.some((policy) => !specification.values.includes(policy))) {
-                    throw new CliUsageError(`${specification.option} for ${command} supports: ${specification.values.join(', ')}.`);
-                }
-            }
-        }
-    }
-}
-function allocatePositionals(command, rawPositionals) {
-    const specifications = commandDefinition(command).grammar.positionals;
-    const maximum = specifications.reduce((sum, specification) => sum + specification.maximum, 0);
-    if (rawPositionals.length > maximum) {
-        throw new CliUsageError(`${command} accepts at most ${maximum} positional value${maximum === 1 ? '' : 's'}.`);
-    }
-    const allocated = {};
-    let offset = 0;
-    for (const [specificationIndex, specification] of specifications.entries()) {
-        const remainingMinimum = specifications.slice(specificationIndex + 1)
-            .reduce((sum, later) => sum + later.minimum, 0);
-        const available = Math.max(0, rawPositionals.length - offset - remainingMinimum);
-        const count = Math.min(specification.maximum, available);
-        const values = Object.freeze(rawPositionals.slice(offset, offset + count));
-        offset += count;
-        if (values.length < specification.minimum) {
-            throw new CliUsageError(`${command} requires ${specification.minimum} ${specification.name} value${specification.minimum === 1 ? '' : 's'}.`);
-        }
-        if (specification.valueKind === 'enum'
-            && values.some((value) => !specification.values.includes(value))) {
-            throw new CliUsageError(`${command} ${specification.name} must be ${humanList(specification.values)}.`);
-        }
-        allocated[specification.name] = values;
-    }
-    return Object.freeze(allocated);
-}
-function present(options, option) {
-    return options.has(option);
-}
-function validateConstraint(command, constraint, options) {
-    if (constraint.kind === 'required') {
-        const missing = constraint.options.filter((option) => !present(options, option));
-        if (missing.length > 0)
-            throw new CliUsageError(`${command} requires ${missing.join(', ')}.`);
-        return;
-    }
-    if (constraint.kind === 'mutually_exclusive') {
-        const selected = constraint.options.filter((option) => present(options, option));
-        if (selected.length > 1) {
-            throw new CliUsageError(`${selected.join(' and ')} are mutually exclusive and may be supplied only once.`);
-        }
-        return;
-    }
-    if (constraint.kind === 'excludes_all') {
-        if (!present(options, constraint.option))
-            return;
-        const selected = constraint.excludedOptions.filter((option) => present(options, option));
-        if (selected.length > 0) {
-            throw new CliUsageError(`${constraint.option} cannot be combined with ${selected.join(', ')}.`);
-        }
-        return;
-    }
-    if (constraint.kind === 'value_excludes') {
-        if (!options.get(constraint.option)?.includes(constraint.value))
-            return;
-        const selected = constraint.excludedOptions.filter((option) => present(options, option));
-        if (selected.length > 0) {
-            throw new CliUsageError(`${constraint.option} ${constraint.value} cannot be combined with ${selected.join(', ')}.`);
-        }
-        return;
-    }
-    if (!present(options, constraint.option))
-        return;
-    const selected = constraint.requiredOptions.filter((option) => present(options, option));
-    if (constraint.kind === 'requires_all' && selected.length !== constraint.requiredOptions.length) {
-        throw new CliUsageError(`${constraint.option} requires ${constraint.requiredOptions.join(' and ')}.`);
-    }
-    if (constraint.kind === 'requires_any' && selected.length === 0) {
-        throw new CliUsageError(`${constraint.option} requires ${constraint.requiredOptions.join(' or ')}.`);
-    }
-}
-function validatePositionalRequirements(command, allocated, options) {
-    for (const specification of commandDefinition(command).grammar.positionals) {
-        if (specification.requiredWhenOptions.some((option) => options.has(option))
-            && (allocated[specification.name]?.length ?? 0) === 0) {
-            throw new CliUsageError(`${specification.requiredWhenOptions.join(' or ')} requires a positional ${specification.name}.`);
-        }
-    }
-}
-function freezeOptions(options) {
-    return new Map([...options].map(([option, values]) => [option, Object.freeze([...values])]));
-}
+import { commandDefinition } from "./command-reference.mjs";
+import { CliGrammarError, parseCliGrammar } from "../packages/analysis/cli-argument-grammar.mjs";
 export function parseCommandArguments(command, argv) {
-    const parsed = parseTokens(command, argv);
-    const options = freezeOptions(parsed.options);
-    validateOptionValues(command, options);
-    const allocated = allocatePositionals(command, parsed.positionals);
-    validatePositionalRequirements(command, allocated, options);
-    for (const constraint of commandDefinition(command).grammar.constraints) {
-        validateConstraint(command, constraint, options);
+    try {
+        return parseCliGrammar(command, commandDefinition(command).grammar, argv);
     }
-    for (const option of ['--config', '--profile']) {
-        const specification = commandDefinition(command).grammar.options.find((candidate) => candidate.option === option);
-        if (specification?.scope === 'common' && options.has(option)) {
-            throw new CliUsageError(`${option} must be resolved before command parsing.`);
-        }
+    catch (cause) {
+        if (cause instanceof CliGrammarError)
+            throw new CliUsageError(cause.message);
+        throw cause;
     }
-    const allPositionals = Object.freeze([...parsed.positionals]);
-    return Object.freeze({
-        command,
-        allPositionals,
-        hasOption: (option) => options.has(option),
-        optionValue: (option) => options.get(option)?.at(-1) ?? null,
-        optionValues: (option) => options.get(option) ?? Object.freeze([]),
-        integerOption: (option) => {
-            const value = options.get(option)?.at(-1);
-            return value === undefined ? null : Number(value);
-        },
-        positionalValue: (name) => allocated[name]?.[0] ?? null,
-        positionalValues: (name) => allocated[name] ?? Object.freeze([]),
-    });
 }