hoi4-agent-tools 3.6.0 → 3.7.0

package/dist/hoi4_agent_tools/reference/script-validation.js

@@ -0,0 +1,277 @@
+import path from 'node:path';
+import { hashCanonical } from '../core/canonical.js';
+import { ServiceError } from '../core/result.js';
+import { locationFor, parseClausewitz, } from '../core/source/index.js';
+const MAX_COMMANDS = 256;
+const booleanCommands = new Set(['AND', 'OR', 'NOT', 'XOR', 'NAND', 'NOR']);
+const containers = new Set([
+    'and',
+    'or',
+    'not',
+    'xor',
+    'nand',
+    'nor',
+    'hidden_trigger',
+    'hidden_effect',
+]);
+const conditionals = new Set(['if', 'else_if', 'else']);
+const iteratorParameters = new Set([
+    'tooltip',
+    'random_select_amount',
+    'display_individual_scopes',
+    'count',
+]);
+function editDistance(a, b) {
+    let row = Array.from({ length: b.length + 1 }, (_, index) => index);
+    for (let i = 0; i < a.length; i++) {
+        const next = [i + 1];
+        for (let j = 0; j < b.length; j++)
+            next.push(Math.min(next[j] + 1, row[j + 1] + 1, row[j] + Number(a[i] !== b[j])));
+        if (Math.min(...next) > 3)
+            return 4;
+        row = next;
+    }
+    return row[b.length];
+}
+/** Check only rules explicitly available from one selected documentation authority. */
+export async function validateScript(references, workspace, input, signal) {
+    if (Buffer.byteLength(input.source, 'utf8') > 64_000)
+        throw new ServiceError('SCRIPT_SOURCE_TOO_LARGE', 'Snippet exceeds 64000 UTF-8 bytes');
+    const inventory = await references.inventory(workspace, signal, [input.documentation]);
+    const catalogs = { effect: new Map(), trigger: new Map() };
+    const sources = new Map();
+    for (const section of inventory.sections) {
+        const filename = path.basename(section.path).toLowerCase();
+        const match = /^(effects?|triggers?)_(?:documentation|docs)\.(?:md|log)$/u.exec(filename);
+        if (match === null)
+            continue;
+        sources.set(section.path, section.revision);
+        if (!/^[a-z_][a-z0-9_]{0,255}$/u.test(section.heading))
+            continue;
+        const scopeLine = /^\s*(?:\*\s*)?Supported Scopes:\s*(.+?)\s*$/imu.exec(section.lines.join('\n'));
+        if (scopeLine === null || !/^[a-z_,\s]+$/iu.test(scopeLine[1]))
+            continue;
+        const scopes = [
+            ...new Set(scopeLine[1]
+                .toLowerCase()
+                .split(/[,\s]+/u)
+                .filter(Boolean)),
+        ].sort();
+        const kind = match[1].startsWith('effect') ? 'effect' : 'trigger';
+        const previous = catalogs[kind].get(section.heading);
+        const { lines: _lines, ...reference } = section;
+        if (previous !== undefined) {
+            previous.conflict ||= previous.scopes.join(',') !== scopes.join(',');
+        }
+        else
+            catalogs[kind].set(section.heading, { reference, scopes, conflict: false });
+    }
+    const document = parseClausewitz(Buffer.from(input.source, 'utf8'), 'snippet');
+    const findings = [];
+    let visited = 0;
+    let checkedCommands = 0;
+    let argumentBlocksUnchecked = 0;
+    let suggestionBudget = 4000;
+    const bindings = new Map();
+    for (const binding of input.bindings) {
+        if (bindings.has(binding.name))
+            throw new ServiceError('SCRIPT_BINDING_DUPLICATE', 'Scope bindings must have unique names');
+        if (binding.name === 'ROOT' || binding.name === 'THIS' || binding.name === 'PREV')
+            throw new ServiceError('SCRIPT_BINDING_RESERVED', 'ROOT, THIS and PREV follow the supplied scope and traversal stack');
+        bindings.set(binding.name, binding.scope);
+    }
+    const report = (entry, kind, scope, details) => {
+        const position = locationFor('snippet', document.lineIndex, entry.start, entry.end).start;
+        findings.push({
+            ...details,
+            command: entry.key.value.slice(0, 256),
+            kind,
+            scope,
+            line: position.line,
+            column: position.column,
+        });
+    };
+    for (const diagnostic of document.diagnostics)
+        findings.push({
+            code: diagnostic.code,
+            status: 'error',
+            command: '',
+            kind: input.kind,
+            scope: input.scope,
+            line: diagnostic.location?.start.line ?? 1,
+            column: diagnostic.location?.start.column ?? 1,
+            message: diagnostic.message.slice(0, 500),
+            suggestions: [],
+        });
+    const walk = (block, kind, scope, previous) => {
+        for (const entry of block.entries) {
+            signal?.throwIfAborted();
+            if (++visited > MAX_COMMANDS) {
+                return;
+            }
+            if (entry.type !== 'assignment') {
+                const position = locationFor('snippet', document.lineIndex, entry.start, entry.end).start;
+                findings.push({
+                    code: 'SCRIPT_EXPECTED_COMMAND',
+                    status: 'error',
+                    command: '',
+                    kind,
+                    scope,
+                    line: position.line,
+                    column: position.column,
+                    message: 'Expected a command assignment in this effect or trigger body',
+                    suggestions: [],
+                });
+                continue;
+            }
+            const raw = entry.key.value;
+            if (raw.startsWith('@'))
+                continue;
+            const name = booleanCommands.has(raw) ? raw.toLowerCase() : raw;
+            const explicitScope = raw === 'ROOT'
+                ? input.scope
+                : raw === 'THIS'
+                    ? scope
+                    : raw === 'PREV'
+                        ? previous
+                        : bindings.get(raw);
+            const fixedScope = /^[A-Z][A-Z0-9]{2}$/u.test(raw) &&
+                !['ROOT', 'THIS', 'PREV', 'FROM', 'AND', 'NOT', 'XOR', 'NOR'].includes(raw)
+                ? 'country'
+                : /^\d+$/u.test(raw)
+                    ? 'state'
+                    : undefined;
+            if (entry.value.type === 'block' &&
+                (explicitScope !== undefined || fixedScope !== undefined)) {
+                walk(entry.value, kind, explicitScope ?? fixedScope, scope);
+                continue;
+            }
+            const signature = catalogs[kind].get(name);
+            if (signature === undefined) {
+                const other = catalogs[kind === 'effect' ? 'trigger' : 'effect'].get(name);
+                const suggestions = name.length > 128
+                    ? []
+                    : [...catalogs[kind].keys()]
+                        .filter((candidate) => Math.abs(candidate.length - name.length) <= 3)
+                        .filter(() => suggestionBudget-- > 0)
+                        .map((candidate) => ({ candidate, distance: editDistance(name, candidate) }))
+                        .filter(({ distance }) => distance <= 3)
+                        .sort((a, b) => a.distance - b.distance || a.candidate.localeCompare(b.candidate, 'en'))
+                        .slice(0, 3)
+                        .map(({ candidate }) => candidate);
+                report(entry, kind, scope, {
+                    code: other === undefined ? 'SCRIPT_COMMAND_UNRESOLVED' : 'SCRIPT_COMMAND_WRONG_KIND',
+                    status: other === undefined ? 'unresolved' : 'error',
+                    message: other === undefined
+                        ? 'Not documented as a native command in the selected source; inspect scripted helpers or declare the scope binding'
+                        : `Documented as a ${kind === 'effect' ? 'trigger' : 'effect'}, not a ${kind}`,
+                    suggestions,
+                    ...(other === undefined ? {} : { reference: other.reference }),
+                });
+                continue;
+            }
+            checkedCommands++;
+            const wrongScope = !signature.scopes.includes('any') && !signature.scopes.includes(scope);
+            report(entry, kind, scope, {
+                code: signature.conflict
+                    ? 'SCRIPT_DOCUMENTATION_CONFLICT'
+                    : wrongScope
+                        ? 'SCRIPT_COMMAND_WRONG_SCOPE'
+                        : 'SCRIPT_COMMAND_SUPPORTED',
+                status: signature.conflict ? 'unresolved' : wrongScope ? 'error' : 'supported',
+                suggestions: [],
+                reference: signature.reference,
+                message: signature.conflict
+                    ? 'Conflicting supported scopes in the selected documentation'
+                    : wrongScope
+                        ? `Supported scopes: ${signature.scopes.join(', ')}`
+                        : 'Command kind and declared scope match the selected documentation',
+            });
+            const iterator = /^(?:every|random|any|all)_(?:(?:owned|controlled|neighbor)_)?(state|country)$/u.exec(name);
+            if (entry.value.type !== 'block') {
+                if (containers.has(name) || conditionals.has(name) || iterator !== null)
+                    report(entry, kind, scope, {
+                        code: 'SCRIPT_EXPECTED_BLOCK',
+                        status: 'error',
+                        message: 'This control command requires a block',
+                        suggestions: [],
+                        reference: signature.reference,
+                    });
+                continue;
+            }
+            if (containers.has(name)) {
+                walk(entry.value, kind, scope, previous);
+                continue;
+            }
+            if (conditionals.has(name) || iterator !== null) {
+                const childScope = iterator?.[1] ?? scope;
+                for (const child of entry.value.entries) {
+                    if (child.type === 'assignment' && child.key.value === 'limit') {
+                        if (child.value.type === 'block')
+                            walk(child.value, 'trigger', childScope, iterator === null ? previous : scope);
+                        else
+                            report(child, 'trigger', childScope, {
+                                code: 'SCRIPT_EXPECTED_BLOCK',
+                                status: 'error',
+                                message: 'limit requires a trigger block',
+                                suggestions: [],
+                            });
+                    }
+                    else if (iterator !== null &&
+                        child.type === 'assignment' &&
+                        iteratorParameters.has(child.key.value)) {
+                        argumentBlocksUnchecked++;
+                    }
+                    else
+                        walk({ type: 'block', entries: [child], start: child.start, end: child.end }, kind, childScope, iterator === null ? previous : scope);
+                }
+            }
+            else {
+                argumentBlocksUnchecked++;
+                report(entry, kind, scope, {
+                    code: 'SCRIPT_ARGUMENT_BLOCK_UNCHECKED',
+                    status: 'unresolved',
+                    message: 'Native argument block is not validated from prose documentation; read the cited syntax before use',
+                    suggestions: [],
+                    reference: signature.reference,
+                });
+            }
+        }
+    };
+    if (document.diagnostics.every(({ severity }) => severity !== 'error' && severity !== 'blocker'))
+        walk(document.root, input.kind, input.scope);
+    const unresolvedCount = findings.filter(({ status }) => status === 'unresolved').length;
+    const invalid = findings.some(({ status }) => status === 'error');
+    const truncated = visited > MAX_COMMANDS;
+    const incomplete = unresolvedCount > 0 || truncated || inventory.skipped > 0 || catalogs[input.kind].size === 0;
+    const order = { error: 0, unresolved: 1, supported: 2 };
+    findings.sort((a, b) => order[a.status] - order[b.status] ||
+        a.line - b.line ||
+        a.column - b.column ||
+        a.code.localeCompare(b.code, 'en'));
+    const result = {
+        valid: invalid ? false : incomplete ? null : true,
+        checksPerformed: ['syntax', 'command_kind', 'declared_scope'],
+        parametersChecked: false,
+        documentation: {
+            source: input.documentation,
+            revision: hashCanonical([...sources].sort()),
+            effectCount: catalogs.effect.size,
+            triggerCount: catalogs.trigger.size,
+            skippedSources: inventory.skipped,
+        },
+        findings: findings.slice(0, input.limit),
+        totalFindings: findings.length,
+        omittedFindings: Math.max(0, findings.length - input.limit),
+        checkedCommands,
+        unresolvedCount,
+        truncated,
+        argumentBlocksUnchecked,
+    };
+    while (Buffer.byteLength(JSON.stringify(result), 'utf8') > 16_000 && result.findings.length > 1)
+        result.findings.pop();
+    result.omittedFindings = result.totalFindings - result.findings.length;
+    if (Buffer.byteLength(JSON.stringify(result), 'utf8') > 16_000)
+        throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Command documentation metadata exceeds the bounded reply');
+    return result;
+}

package/dist/hoi4_agent_tools/reference/service.js

@@ -1,15 +1,29 @@
-import { open, realpath, stat } from 'node:fs/promises';
+import { realpath } from 'node:fs/promises';
 import path from 'node:path';
-import fg from 'fast-glob';
+import { globFiles } from '../core/file-glob.js';
 import { compareCodeUnits, sha256Bytes } from '../core/canonical.js';
 import { ServiceError } from '../core/result.js';
 import { isWithin } from '../core/workspace.js';
-import { readBoundedLines } from './lines.js';
-import { readBoundedFile } from '../core/scanner.js';
+import { prepareSource, verifySource, readBoundedFile, } from '../core/scanner.js';
+import { decodeSource } from '../core/source/encoding.js';
+import { readBoundedJsonLines } from './lines.js';
 const MAX_FILES = 512;
 const MAX_FILE_BYTES = 2_000_000;
 const MAX_TOTAL_BYTES = 32_000_000;
 const MAX_SECTIONS = 20_000;
+const MAX_REFERENCE_METADATA_BYTES = 24_000;
+function referenceText(bytes) {
+    if (bytes.length >= 2 &&
+        ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff))) {
+        const payload = Buffer.from(bytes.subarray(2));
+        if (payload.length % 2 !== 0)
+            throw new ServiceError('REFERENCE_ENCODING_INVALID', 'UTF-16 reference text has an incomplete code unit');
+        if (bytes[0] === 0xfe)
+            payload.swap16();
+        return payload.toString('utf16le');
+    }
+    return decodeSource(bytes).text;
+}
 const searchStopWords = new Set([
     'a',
     'an',
@@ -78,7 +92,7 @@ function sectionize(root, filePath, content, revision) {
     const relative = path.relative(root.root, filePath).replaceAll('\\', '/');
     const title = path
         .basename(filePath)
-        .replace(/\.(md|log)$/u, '')
+        .replace(/\.(md|log)$/iu, '')
         .replaceAll('_', ' ')
         .replace(/ - Hearts of Iron 4 Wiki$/u, '');
     const headings = [];
@@ -95,7 +109,7 @@ function sectionize(root, filePath, content, revision) {
         if (match)
             headings.push({ heading: match[2].replace(/<a\b[^>]*><\/a>/gu, '').trim(), start: index });
         else if (root.kind === 'script_doc' &&
-            filePath.endsWith('.log') &&
+            filePath.toLowerCase().endsWith('.log') &&
             /^[A-Za-z_][A-Za-z0-9_.:-]*$/u.test(line.trim()) &&
             !/^\s/u.test(line) &&
             /^\s+\S/u.test(lines[index + 1] ?? '')) {
@@ -222,16 +236,33 @@ export class ReferenceService {
             this.#cacheBytes -= oldest[1].bytes;
         }
     }
-    async inventory(workspace, signal) {
+    async inventory(workspace, signal, sources) {
         const sections = [];
         const coverage = {
-            game_doc: { files: 0, sections: 0, unavailable: true },
-            wiki: { files: 0, sections: 0, unavailable: true },
-            script_doc: { files: 0, sections: 0, unavailable: true },
+            game_doc: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('game_doc'),
+            },
+            wiki: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('wiki'),
+            },
+            script_doc: {
+                files: 0,
+                sections: 0,
+                unavailable: true,
+                included: sources === undefined || sources.includes('script_doc'),
+            },
         };
         let skipped = 0;
         let bytes = 0;
         for (const root of sourceRoots(workspace)) {
+            if (sources !== undefined && !sources.includes(root.kind))
+                continue;
             signal?.throwIfAborted();
             let canonicalRoot;
             try {
@@ -250,80 +281,104 @@ export class ReferenceService {
                 throw new ServiceError('REFERENCE_ROOT_ESCAPE', 'Reference root resolves outside its configured workspace');
             }
             coverage[root.kind].unavailable = false;
-            const names = await fg(root.patterns, {
+            const names = await globFiles(root.patterns, {
                 cwd: canonicalRoot,
-                onlyFiles: true,
-                followSymbolicLinks: false,
-                dot: false,
+                caseSensitive: false,
+                ...(signal === undefined ? {} : { signal }),
             });
             names.sort(compareCodeUnits);
-            for (const name of names) {
+            for (let offset = 0; offset < names.length; offset += 8) {
                 signal?.throwIfAborted();
                 if (coverage[root.kind].files >= MAX_FILES || sections.length >= MAX_SECTIONS) {
-                    skipped++;
-                    continue;
-                }
-                const candidate = path.join(canonicalRoot, name);
-                const canonical = await realpath(candidate);
-                if (!isWithin(canonicalRoot, canonical)) {
-                    skipped++;
-                    continue;
-                }
-                const metadata = await stat(canonical);
-                if (metadata.size > MAX_FILE_BYTES || bytes + metadata.size > MAX_TOTAL_BYTES) {
-                    skipped++;
-                    continue;
+                    skipped += names.length - offset;
+                    break;
                 }
-                bytes += metadata.size;
-                coverage[root.kind].files++;
-                const key = `${root.kind}:${canonicalRoot}:${canonical}`;
-                const handle = await open(canonical, 'r');
-                let content;
+                const readRoot = { path: canonicalRoot };
+                const prepared = await Promise.allSettled(names.slice(offset, offset + 8).map((name) => prepareSource(readRoot, name, signal)));
+                const opened = prepared.flatMap((entry) => entry.status === 'fulfilled' ? [entry.value] : []);
                 try {
-                    const opened = await handle.stat();
-                    const resolvedAfterOpen = await realpath(canonical);
-                    const current = await stat(resolvedAfterOpen);
-                    if (!opened.isFile() ||
-                        !isWithin(canonicalRoot, resolvedAfterOpen) ||
-                        opened.dev !== current.dev ||
-                        opened.ino !== current.ino) {
+                    signal?.throwIfAborted();
+                    for (const entry of prepared) {
+                        if (entry.status !== 'rejected')
+                            continue;
+                        if (entry.reason instanceof ServiceError && entry.reason.code === 'SCAN_ROOT_ESCAPE') {
+                            skipped++;
+                            continue;
+                        }
                         throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference path changed while opening it; retry the request');
                     }
-                    content = await readBoundedFile(handle, Math.min(MAX_FILE_BYTES, MAX_TOTAL_BYTES - bytes + metadata.size), signal);
+                    const admittedSources = [];
+                    for (const source of opened) {
+                        if (!source.metadata.isFile() ||
+                            source.metadata.size > MAX_FILE_BYTES ||
+                            bytes + source.metadata.size > MAX_TOTAL_BYTES ||
+                            coverage[root.kind].files >= MAX_FILES) {
+                            skipped++;
+                            continue;
+                        }
+                        bytes += source.metadata.size;
+                        coverage[root.kind].files++;
+                        admittedSources.push(source);
+                    }
+                    const reads = await Promise.allSettled(admittedSources.map(async (source) => {
+                        const content = await readBoundedFile(source.handle, source.metadata.size, signal);
+                        await verifySource(readRoot, source);
+                        if (content.length !== source.metadata.size)
+                            throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference changed while it was being read; retry the request');
+                        return content;
+                    }));
+                    signal?.throwIfAborted();
+                    const failed = reads.find((entry) => entry.status === 'rejected');
+                    if (failed?.status === 'rejected')
+                        throw new ServiceError('REFERENCE_SOURCE_CHANGED', 'Reference changed while it was being read; retry the request');
+                    for (const [index, read] of reads.entries()) {
+                        if (read.status !== 'fulfilled')
+                            continue;
+                        const canonical = admittedSources[index].canonicalPath;
+                        const content = read.value;
+                        const key = `${root.kind}:${canonicalRoot}:${canonical}`;
+                        const revision = sha256Bytes(content);
+                        let cached = this.#cache.get(key);
+                        if (cached?.revision !== revision)
+                            cached = {
+                                revision,
+                                bytes: content.length,
+                                sections: sectionize(root, canonical, referenceText(content), revision),
+                            };
+                        this.retain(key, cached);
+                        const admitted = cached.sections.slice(0, MAX_SECTIONS - sections.length);
+                        if (admitted.length < cached.sections.length)
+                            skipped++;
+                        sections.push(...admitted);
+                        coverage[root.kind].sections += admitted.length;
+                    }
                 }
                 finally {
-                    await handle.close();
+                    await Promise.all(opened.map(({ handle }) => handle.close()));
                 }
-                bytes += content.length - metadata.size;
-                const revision = sha256Bytes(content);
-                let cached = this.#cache.get(key);
-                if (cached?.revision !== revision) {
-                    cached = {
-                        revision,
-                        bytes: content.length,
-                        sections: sectionize(root, canonical, content.toString('utf8'), revision),
-                    };
-                }
-                this.retain(key, cached);
-                const admitted = cached.sections.slice(0, MAX_SECTIONS - sections.length);
-                if (admitted.length < cached.sections.length)
-                    skipped++;
-                sections.push(...admitted);
-                coverage[root.kind].sections += admitted.length;
             }
         }
         return { sections, coverage, skipped };
     }
     async search(workspace, input, signal) {
-        const inventory = await this.inventory(workspace, signal);
+        const inventory = await this.inventory(workspace, signal, input.sources);
         const matches = this.rank(inventory.sections, input.query, input.sources);
+        const results = matches.slice(0, input.limit).map(({ section, score, excerpt }) => ({
+            ...this.publicSection(section),
+            score,
+            excerpt,
+        }));
+        const requested = results.length;
+        while (Buffer.byteLength(JSON.stringify(results), 'utf8') > MAX_REFERENCE_METADATA_BYTES &&
+            results.length > 1)
+            results.pop();
+        if (Buffer.byteLength(JSON.stringify(results), 'utf8') > MAX_REFERENCE_METADATA_BYTES)
+            throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Reference metadata exceeds the bounded reply');
         return {
-            results: matches.slice(0, input.limit).map(({ section, score, excerpt }) => ({
-                ...this.publicSection(section),
-                score,
-                excerpt,
-            })),
+            results,
             total: matches.length,
+            omitted: matches.length - results.length,
+            limitedByBytes: results.length < requested,
             coverage: inventory.coverage,
             skipped: inventory.skipped,
         };
@@ -364,7 +419,17 @@ export class ReferenceService {
             left.section.startLine - right.section.startLine);
     }
     async read(workspace, input, signal) {
-        const inventory = await this.inventory(workspace, signal);
+        let source = input.source;
+        if (source === undefined) {
+            for (const cached of this.#cache.values()) {
+                const section = cached.sections.find(({ id }) => id === input.id);
+                if (section !== undefined) {
+                    source = section.source;
+                    break;
+                }
+            }
+        }
+        const inventory = await this.inventory(workspace, signal, source === undefined ? undefined : [source]);
         const section = inventory.sections.find(({ id }) => id === input.id);
         if (section === undefined)
             throw new ServiceError('REFERENCE_SECTION_UNKNOWN', 'Reference section is not in this workspace');
@@ -373,7 +438,7 @@ export class ReferenceService {
                 currentRevision: section.revision,
             });
         const startLine = input.startLine ?? section.startLine;
-        const selected = readBoundedLines(section.lines, section.startLine, startLine, input.startColumn ?? 1, input.maxLines, 8_000);
+        const selected = readBoundedJsonLines(section.lines, section.startLine, startLine, input.startColumn ?? 1, input.maxLines, 8_000, 8_192);
         return {
             ...this.publicSection(section),
             sectionStartLine: section.startLine,
@@ -422,13 +487,20 @@ export class ReferenceService {
             return true;
         });
         const selected = prioritized.slice(0, input.limit);
+        const requestedCount = selected.length;
+        while (Buffer.byteLength(JSON.stringify(selected.map((section) => this.publicSection(section))), 'utf8') > MAX_REFERENCE_METADATA_BYTES &&
+            selected.length > 1)
+            selected.pop();
+        if (Buffer.byteLength(JSON.stringify(selected.map((section) => this.publicSection(section))), 'utf8') > MAX_REFERENCE_METADATA_BYTES)
+            throw new ServiceError('REFERENCE_RESPONSE_TOO_LARGE', 'Reference metadata exceeds the bounded reply');
         return {
             surface: input.surface,
             sections: selected.map((section) => ({
                 ...this.publicSection(section),
                 excerpt: ranked.find((entry) => entry.section.id === section.id)?.excerpt ?? section.excerpt,
             })),
-            omitted: Math.max(0, prioritized.length - input.limit),
+            omitted: Math.max(0, prioritized.length - selected.length),
+            limitedByBytes: selected.length < requestedCount,
             omittedSources: required
                 .filter(({ section }) => !selected.some((entry) => entry.path === section.path && entry.source === section.source))
                 .map(({ name }) => name),