knock 0.1.0 → 1.0.0

package/dist/index.js

@@ -0,0 +1,278 @@
+/**
+ * knock — subdomain enumeration via Certificate Transparency logs and
+ * wordlist DNS brute forcing, with wildcard DNS detection.
+ */
+import { Resolver } from 'node:dns/promises';
+import { readFile } from 'node:fs/promises';
+import { randomBytes } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { loadPublicSuffixList, getRegistrableDomain } from './psl.js';
+/** Absolute paths to the wordlists bundled with the package. */
+export const wordlists = {
+    subs: fileURLToPath(new URL('../lists/subs.txt', import.meta.url)),
+    org: fileURLToPath(new URL('../lists/org.txt', import.meta.url)),
+};
+// DNS answers that simply mean "nothing there", as opposed to a lookup failure.
+const NEGATIVE_CODES = new Set(['ENOTFOUND', 'ENODATA', 'NXDOMAIN']);
+const HOSTNAME_PATTERN = /^[a-z0-9_][a-z0-9_.-]*$/;
+// Dotted-quad IPv4 literals have no registrable domain; skip PSL reduction.
+const IPV4_PATTERN = /^\d{1,3}(?:\.\d{1,3}){3}$/;
+const defaults = {
+    wordlist: wordlists.subs,
+    concurrency: 64,
+    timeout: 5000,
+    tries: 2,
+    family: 4,
+    wildcardTests: 3,
+    ct: true,
+    ctTimeout: 15000,
+    verify: true,
+    web: false,
+    webTimeout: 5000,
+    baseDomainOnly: true,
+    icannOnly: false,
+};
+/**
+ * Reduce user input like "HTTPS://www.Example.com/path" to a bare hostname.
+ * Throws TypeError when no usable hostname can be extracted.
+ */
+export function normalizeDomain(input) {
+    const raw = String(input ?? '').trim();
+    if (raw === '')
+        throw new TypeError('domain is required');
+    const url = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `http://${raw}`;
+    let hostname;
+    try {
+        hostname = new URL(url).hostname;
+    }
+    catch {
+        throw new TypeError(`invalid domain: ${raw}`);
+    }
+    hostname = hostname.toLowerCase().replace(/\.+$/, '');
+    if (hostname === '' || hostname.startsWith('[')) {
+        throw new TypeError(`invalid domain: ${raw}`);
+    }
+    return hostname;
+}
+/** Parse wordlist text into unique, lowercased labels. Blank lines and #comments are skipped. */
+export function parseWordlist(text) {
+    const words = new Set();
+    for (const line of String(text).split(/\r?\n/)) {
+        const word = line.trim().toLowerCase();
+        if (word === '' || word.startsWith('#'))
+            continue;
+        words.add(word);
+    }
+    return [...words];
+}
+/** Read and parse a wordlist file. */
+export async function loadWordlist(path) {
+    return parseWordlist(await readFile(path, 'utf8'));
+}
+/**
+ * Query Certificate Transparency logs (via crt.sh) for names certified under
+ * `domain`. Purely passive: one HTTPS request, no DNS traffic to the target.
+ * Wildcard entries like "*.dev.example.com" are reported as "dev.example.com".
+ * Resolves to a sorted array of hostnames, the apex excluded.
+ */
+export async function certSubdomains(domain, options = {}) {
+    const target = normalizeDomain(domain);
+    const { timeout = defaults.ctTimeout, signal, fetch: fetchImpl = globalThis.fetch } = options;
+    const url = `https://crt.sh/?q=${encodeURIComponent(`%.${target}`)}&output=json`;
+    const signals = [AbortSignal.timeout(timeout)];
+    if (signal)
+        signals.push(signal);
+    const response = await fetchImpl(url, {
+        signal: AbortSignal.any(signals),
+        headers: { accept: 'application/json' },
+    });
+    if (!response.ok)
+        throw new Error(`crt.sh responded with HTTP ${response.status}`);
+    const entries = await response.json();
+    const names = new Set();
+    for (const entry of Array.isArray(entries) ? entries : []) {
+        const record = entry;
+        const candidates = [record?.common_name, ...String(record?.name_value ?? '').split('\n')];
+        for (const candidate of candidates) {
+            if (!candidate)
+                continue;
+            let name = String(candidate).trim().toLowerCase().replace(/\.+$/, '');
+            if (name.startsWith('*.'))
+                name = name.slice(2);
+            if (name === target || !name.endsWith(`.${target}`))
+                continue;
+            if (!HOSTNAME_PATTERN.test(name))
+                continue;
+            names.add(name);
+        }
+    }
+    return [...names].sort();
+}
+function makeResolver({ timeout, tries, servers, }) {
+    const resolver = new Resolver({ timeout, tries });
+    if (servers && servers.length > 0)
+        resolver.setServers(servers);
+    return resolver;
+}
+async function lookupName(resolver, name, family) {
+    const addresses = [];
+    let error;
+    const families = family === 'any' ? [4, 6] : [family];
+    for (const fam of families) {
+        try {
+            const found = fam === 6 ? await resolver.resolve6(name) : await resolver.resolve4(name);
+            addresses.push(...found);
+        }
+        catch (err) {
+            const code = err?.code;
+            if (!NEGATIVE_CODES.has(code ?? ''))
+                error = err;
+        }
+    }
+    return { addresses, error };
+}
+// Resolve a few random labels that cannot exist; any answers reveal a wildcard record.
+async function detectWildcard(resolver, domain, { tests, family }) {
+    const addresses = new Set();
+    for (let i = 0; i < tests; i += 1) {
+        const name = `knock-${randomBytes(8).toString('hex')}.${domain}`;
+        const { addresses: found } = await lookupName(resolver, name, family);
+        for (const address of found)
+            addresses.add(address);
+    }
+    return addresses;
+}
+async function probeWeb(name, timeout) {
+    const statuses = { https: null, http: null };
+    await Promise.all(['https', 'http'].map(async (protocol) => {
+        try {
+            const response = await fetch(`${protocol}://${name}/`, {
+                method: 'HEAD',
+                redirect: 'manual',
+                signal: AbortSignal.timeout(timeout),
+            });
+            statuses[protocol] = response.status;
+        }
+        catch {
+            statuses[protocol] = null;
+        }
+    }));
+    return statuses;
+}
+async function pool(items, worker, concurrency, signal) {
+    let index = 0;
+    const size = Math.max(1, Math.min(concurrency, items.length));
+    const runners = Array.from({ length: size }, async () => {
+        while (index < items.length && !signal?.aborted) {
+            const item = items[index];
+            index += 1;
+            await worker(item);
+        }
+    });
+    await Promise.all(runners);
+}
+/**
+ * Enumerate subdomains of `domain`.
+ *
+ * Two sources feed the candidate set: Certificate Transparency logs
+ * (passive, `ct` option) and a wordlist brute force over DNS. Candidates
+ * are resolved concurrently; hosts that only echo a wildcard DNS answer are
+ * suppressed unless Certificate Transparency vouches for them.
+ */
+export async function knock(domain, options = {}) {
+    const opts = { ...defaults, ...options };
+    let target = normalizeDomain(domain);
+    if (opts.baseDomainOnly && !IPV4_PATTERN.test(target)) {
+        const rules = opts.psl ?? (await loadPublicSuffixList());
+        const base = getRegistrableDomain(target, rules, { icannOnly: opts.icannOnly });
+        if (base)
+            target = base;
+    }
+    const words = opts.words ?? (await loadWordlist(opts.wordlist));
+    const resolver = opts.resolver ?? makeResolver(opts);
+    const { signal } = opts;
+    signal?.throwIfAborted();
+    const started = Date.now();
+    const ct = { enabled: Boolean(opts.ct), names: 0, unresolved: [], error: null };
+    const candidates = new Map();
+    for (const word of words) {
+        candidates.set(`${word}.${target}`, new Set(['wordlist']));
+    }
+    if (opts.ct) {
+        try {
+            const names = await certSubdomains(target, {
+                timeout: opts.ctTimeout,
+                signal,
+                fetch: opts.fetch,
+            });
+            ct.names = names.length;
+            for (const name of names) {
+                const sources = candidates.get(name);
+                if (sources)
+                    sources.add('ct');
+                else
+                    candidates.set(name, new Set(['ct']));
+            }
+        }
+        catch (err) {
+            if (signal?.aborted)
+                throw err;
+            ct.error = err instanceof Error ? err.message : String(err);
+        }
+    }
+    const wildcardAddresses = words.length > 0 && opts.wildcardTests > 0
+        ? await detectWildcard(resolver, target, { tests: opts.wildcardTests, family: opts.family })
+        : new Set();
+    const results = [];
+    let queried = 0;
+    let errors = 0;
+    await pool([...candidates.entries()], async ([name, sources]) => {
+        const fromCt = sources.has('ct');
+        const emit = async (addresses) => {
+            const result = { name, addresses, sources: [...sources] };
+            if (opts.web)
+                result.web = await probeWeb(name, opts.webTimeout);
+            results.push(result);
+            opts.onResult?.(result);
+        };
+        if (fromCt && !sources.has('wordlist') && !opts.verify) {
+            await emit([]);
+            return;
+        }
+        const { addresses, error } = await lookupName(resolver, name, opts.family);
+        queried += 1;
+        if (addresses.length === 0) {
+            if (error)
+                errors += 1;
+            if (fromCt)
+                ct.unresolved.push(name);
+            return;
+        }
+        // A wordlist hit that only mirrors the wildcard answer is not a real
+        // discovery; a certificate in the CT logs is evidence on its own.
+        if (!fromCt &&
+            wildcardAddresses.size > 0 &&
+            addresses.every((address) => wildcardAddresses.has(address))) {
+            return;
+        }
+        await emit(addresses);
+    }, opts.concurrency, signal);
+    signal?.throwIfAborted();
+    ct.unresolved.sort();
+    return {
+        domain: target,
+        wildcard: { detected: wildcardAddresses.size > 0, addresses: [...wildcardAddresses] },
+        ct,
+        results,
+        stats: {
+            words: words.length,
+            candidates: candidates.size,
+            queried,
+            found: results.length,
+            errors,
+            durationMs: Date.now() - started,
+        },
+    };
+}
+export { publicSuffixListPath, parsePublicSuffixList, loadPublicSuffixList, getPublicSuffix, getRegistrableDomain, } from './psl.js';
+export default knock;

package/dist/psl.d.ts

@@ -0,0 +1,36 @@
+/** Absolute path to the vendored Public Suffix List. */
+export declare const publicSuffixListPath: string;
+/** A single Public Suffix List rule. */
+export interface PublicSuffixRule {
+    /** The rule began with `!`, marking an exception to a wildcard rule. */
+    exception: boolean;
+    /** From the ICANN section rather than the PRIVATE section of the list. */
+    icann: boolean;
+}
+/** Parsed rule set: punycode rule string (without any leading `!`) → rule. */
+export type PublicSuffixRules = Map<string, PublicSuffixRule>;
+/** Options shared by the Public Suffix List lookups. */
+export interface PublicSuffixOptions {
+    /** Ignore the PRIVATE section (github.io, herokuapp.com, …). Default false. */
+    icannOnly?: boolean;
+}
+/**
+ * Parse Public Suffix List text into a rule map. Each line is read only up to
+ * the first whitespace; `//` lines and blanks are skipped. The ICANN/PRIVATE
+ * section is tracked from the `===BEGIN …===` markers.
+ */
+export declare function parsePublicSuffixList(text: string): PublicSuffixRules;
+/** Read and parse a Public Suffix List file, memoized per path. */
+export declare function loadPublicSuffixList(path?: string): Promise<PublicSuffixRules>;
+/**
+ * The public suffix of `hostname` (e.g. "co.uk", "github.io"), or null when the
+ * hostname has fewer labels than the matching rule. `hostname` should already
+ * be a bare, lowercased, punycode host as produced by `normalizeDomain`.
+ */
+export declare function getPublicSuffix(hostname: string, rules: PublicSuffixRules, options?: PublicSuffixOptions): string | null;
+/**
+ * The registrable domain of `hostname` — its public suffix plus one label
+ * (e.g. "www.example.co.uk" → "example.co.uk"). Returns null when `hostname` is
+ * itself a public suffix (or shorter), i.e. has no registrable domain.
+ */
+export declare function getRegistrableDomain(hostname: string, rules: PublicSuffixRules, options?: PublicSuffixOptions): string | null;

package/dist/psl.js

@@ -0,0 +1,125 @@
+/**
+ * Public Suffix List support — reduce a hostname to its registrable domain.
+ *
+ * Where a name may be registered (`co.uk`, `github.io`, `s3.amazonaws.com`)
+ * cannot be inferred from string rules alone, so knock uses the Public Suffix
+ * List. The data is vendored in `lists/public_suffix_list.dat`; refresh it with
+ * `npm run update-psl`.
+ *
+ * @see https://github.com/publicsuffix/list/wiki/Format for the rule format and
+ *   the matching algorithm implemented in {@link publicSuffixLength}.
+ */
+import { readFile } from 'node:fs/promises';
+import { domainToASCII, fileURLToPath } from 'node:url';
+/** Absolute path to the vendored Public Suffix List. */
+export const publicSuffixListPath = fileURLToPath(new URL('../lists/public_suffix_list.dat', import.meta.url));
+// Punycode a rule so it matches the ASCII hostnames the WHATWG URL parser
+// produces (the list is UTF-8; hostnames come through as xn--…). A leftmost
+// `*` wildcard is preserved.
+function toAscii(name) {
+    if (name === '*')
+        return '*';
+    if (name.startsWith('*.')) {
+        const rest = domainToASCII(name.slice(2));
+        return rest ? `*.${rest}` : name.toLowerCase();
+    }
+    return domainToASCII(name) || name.toLowerCase();
+}
+/**
+ * Parse Public Suffix List text into a rule map. Each line is read only up to
+ * the first whitespace; `//` lines and blanks are skipped. The ICANN/PRIVATE
+ * section is tracked from the `===BEGIN …===` markers.
+ */
+export function parsePublicSuffixList(text) {
+    const rules = new Map();
+    let icann = true;
+    for (const rawLine of String(text).split(/\r?\n/)) {
+        if (rawLine.includes('===BEGIN ICANN DOMAINS===')) {
+            icann = true;
+            continue;
+        }
+        if (rawLine.includes('===BEGIN PRIVATE DOMAINS===')) {
+            icann = false;
+            continue;
+        }
+        const token = rawLine.trim().split(/\s+/)[0] ?? '';
+        if (token === '' || token.startsWith('//'))
+            continue;
+        const exception = token.startsWith('!');
+        rules.set(toAscii(exception ? token.slice(1) : token), { exception, icann });
+    }
+    return rules;
+}
+const cache = new Map();
+/** Read and parse a Public Suffix List file, memoized per path. */
+export function loadPublicSuffixList(path = publicSuffixListPath) {
+    let pending = cache.get(path);
+    if (!pending) {
+        pending = readFile(path, 'utf8')
+            .then(parsePublicSuffixList)
+            .catch((error) => {
+            cache.delete(path);
+            throw error;
+        });
+        cache.set(path, pending);
+    }
+    return pending;
+}
+// Number of labels in the public suffix of `labels`, per the PSL algorithm:
+// an exception rule prevails over any wildcard/normal rule; otherwise the rule
+// with the most labels prevails; with no match the default rule is `*` (one
+// label). An exception rule's suffix is one label shorter than the rule.
+function publicSuffixLength(labels, rules, icannOnly) {
+    let best = null;
+    for (let i = 0; i < labels.length; i += 1) {
+        const length = labels.length - i;
+        const exact = labels.slice(i).join('.');
+        const wildcard = ['*', ...labels.slice(i + 1)].join('.');
+        for (const key of exact === wildcard ? [exact] : [exact, wildcard]) {
+            const rule = rules.get(key);
+            if (!rule || (icannOnly && !rule.icann))
+                continue;
+            if (!best ||
+                (rule.exception && !best.exception) ||
+                (rule.exception === best.exception && length > best.length)) {
+                best = { exception: rule.exception, length };
+            }
+        }
+    }
+    if (!best)
+        return 1;
+    return best.exception ? best.length - 1 : best.length;
+}
+function cleanHost(hostname) {
+    return String(hostname).toLowerCase().replace(/\.+$/, '');
+}
+/**
+ * The public suffix of `hostname` (e.g. "co.uk", "github.io"), or null when the
+ * hostname has fewer labels than the matching rule. `hostname` should already
+ * be a bare, lowercased, punycode host as produced by `normalizeDomain`.
+ */
+export function getPublicSuffix(hostname, rules, options = {}) {
+    const host = cleanHost(hostname);
+    if (host === '')
+        return null;
+    const labels = host.split('.');
+    const length = publicSuffixLength(labels, rules, options.icannOnly ?? false);
+    if (length > labels.length)
+        return null;
+    return labels.slice(labels.length - length).join('.');
+}
+/**
+ * The registrable domain of `hostname` — its public suffix plus one label
+ * (e.g. "www.example.co.uk" → "example.co.uk"). Returns null when `hostname` is
+ * itself a public suffix (or shorter), i.e. has no registrable domain.
+ */
+export function getRegistrableDomain(hostname, rules, options = {}) {
+    const host = cleanHost(hostname);
+    if (host === '')
+        return null;
+    const labels = host.split('.');
+    const length = publicSuffixLength(labels, rules, options.icannOnly ?? false);
+    if (labels.length <= length)
+        return null;
+    return labels.slice(labels.length - length - 1).join('.');
+}