@withtruing/companion 0.13.3 → 0.14.1

package/dist/whole-read/packing.js

@@ -0,0 +1,612 @@
+import { createHash } from 'node:crypto';
+import { decodeCanonicalBlob } from './decode.js';
+import { redactCanonicalText } from './redaction.js';
+import { sanitizeBuildMetadata } from './sanitize.js';
+import { WHOLE_READ_PHASE_POLICY } from './phase-policy.js';
+export const READ_PLANNED_INPUT_TOKENS_PER_CHARACTER = 0.5;
+export const READ_OUTPUT_TOKENS_PER_INPUT_TOKEN = 0.6;
+export const MIN_WHOLE_READ_OUTPUT_TOKENS = 2_048;
+export const MAX_WHOLE_READ_OUTPUT_TOKENS = WHOLE_READ_PHASE_POLICY.read.maxOutputTokens;
+/** At 0.5 planned tokens per character this stays below Haiku's 100k lower-price prompt tier. */
+export const MAX_WHOLE_READ_PROMPT_CHARACTERS = 106_666;
+export const WHOLE_READ_PACKING_VERSION = 'whole-read-packing@4';
+export function plannedReadInputTokens(promptCharacters) {
+    if (!Number.isSafeInteger(promptCharacters) || promptCharacters <= 0)
+        return invalid();
+    return Math.ceil(promptCharacters * READ_PLANNED_INPUT_TOKENS_PER_CHARACTER);
+}
+export function readUnitOutputTokens(promptCharacters) {
+    return Math.min(MAX_WHOLE_READ_OUTPUT_TOKENS, Math.max(MIN_WHOLE_READ_OUTPUT_TOKENS, Math.ceil(plannedReadInputTokens(promptCharacters) * READ_OUTPUT_TOKENS_PER_INPUT_TOKEN)));
+}
+const invalid = () => { throw new TypeError('whole_read_plan_invalid'); };
+const hash = (value) => `sha256:${createHash('sha256').update(value, 'utf8').digest('hex')}`;
+function crlfBoundaries(source) {
+    const result = new Set();
+    for (const line of source.decoded.mapping.lines)
+        if (line.endCharacter - line.contentEndCharacter === 2) {
+            result.add(source.decoded.mapping.byteOffsetAtCharacter(line.contentEndCharacter + 1));
+        }
+    return result;
+}
+function prepare(source) {
+    if (!source.path || !sanitizeBuildMetadata({ path: source.path }).ok)
+        return invalid();
+    const redacted = redactCanonicalText({ text: source.decoded.text });
+    if (redacted.mapping.segments.every((segment) => segment.kind === 'source')
+        && redacted.text === source.decoded.text)
+        return {
+            source, text: source.decoded.text, codeUnits: source.decoded.mapping.characterCodeUnitOffsets,
+            boundaries: source.decoded.mapping.canonicalByteOffsets.map((byte, delivered) => ({ byte, delivered })),
+            lines: source.decoded.mapping.lines, forbiddenBytes: crlfBoundaries(source), replacementBreaks: [],
+            hasDeliveredBoundary: () => true,
+            deliveredSpan: (startByte, endByte) => [source.decoded.mapping.characterOffsetAtByte(startByte),
+                source.decoded.mapping.characterOffsetAtByte(endByte)],
+        };
+    const bomLength = source.decoded.bom?.endByte ?? 0;
+    const utf8 = source.decoded.encoding === 'utf8' ? undefined
+        : decodeCanonicalBlob(new TextEncoder().encode(source.decoded.text));
+    if (utf8 && utf8.status !== 'decoded')
+        return invalid();
+    const toActual = (utf8Byte) => source.decoded.encoding === 'utf8'
+        ? utf8Byte + bomLength
+        : source.decoded.mapping.byteOffsetAtCharacter(utf8.mapping.characterOffsetAtByte(utf8Byte));
+    const lineBreakCounts = [0];
+    for (const character of source.decoded.text) {
+        lineBreakCounts.push(lineBreakCounts.at(-1) + (character === '\n' ? 1 : 0));
+    }
+    const replacementBreaks = redacted.mapping.segments.flatMap((segment) => {
+        if (segment.kind !== 'replacement')
+            return [];
+        const start = source.decoded.mapping.characterOffsetAtByte(toActual(segment.canonicalStartByte));
+        const end = source.decoded.mapping.characterOffsetAtByte(toActual(segment.canonicalEndByte));
+        const count = lineBreakCounts[end] - lineBreakCounts[start];
+        return count ? [{ deliveredAfter: segment.deliveredEnd, count }] : [];
+    });
+    const boundaries = [];
+    const add = (point) => {
+        const previous = boundaries.at(-1);
+        if (previous?.delivered === point.delivered)
+            boundaries[boundaries.length - 1] = point;
+        else
+            boundaries.push(point);
+    };
+    for (const segment of redacted.mapping.segments) {
+        if (segment.kind === 'source')
+            segment.canonicalByteOffsets.forEach((byte, index) => add({
+                delivered: segment.deliveredStart + index, byte: toActual(byte),
+            }));
+        else {
+            add({ delivered: segment.deliveredStart, byte: toActual(segment.canonicalStartByte) });
+            add({ delivered: segment.deliveredEnd, byte: toActual(segment.canonicalEndByte) });
+        }
+    }
+    if (!boundaries.length) {
+        const start = source.decoded.bom?.endByte ?? 0;
+        boundaries.push({ delivered: 0, byte: start });
+    }
+    const deliveredSpan = (startByte, endByte) => {
+        const start = boundaries.find((point) => point.byte === startByte);
+        const end = boundaries.findLast((point) => point.byte === endByte);
+        if (!start || !end || start.delivered > end.delivered)
+            return invalid();
+        return [start.delivered, end.delivered];
+    };
+    const hasDeliveredBoundary = (byte) => {
+        let low = 0;
+        let high = boundaries.length - 1;
+        while (low <= high) {
+            const middle = (low + high) >>> 1;
+            const point = boundaries[middle].byte;
+            if (point === byte)
+                return true;
+            if (point < byte)
+                low = middle + 1;
+            else
+                high = middle - 1;
+        }
+        return false;
+    };
+    const forbiddenBytes = crlfBoundaries(source);
+    const codeUnits = [0];
+    let codeUnit = 0;
+    for (const character of redacted.text) {
+        codeUnit += character.length;
+        codeUnits.push(codeUnit);
+    }
+    return { source, text: redacted.text, codeUnits, boundaries, lines: source.decoded.mapping.lines,
+        forbiddenBytes, replacementBreaks, deliveredSpan, hasDeliveredBoundary };
+}
+/** Recreates the exact redacted text slice used while packing a canonical byte span. */
+export function deliveredCanonicalSlice(decoded, startByte, endByte) {
+    const prepared = prepare({ entryId: 'delivery', path: 'delivery', blobId: 'delivery',
+        inventoryRevision: 1, decoded });
+    const [start, end] = prepared.deliveredSpan(startByte, endByte);
+    return prepared.text.slice(prepared.codeUnits[start], prepared.codeUnits[end]);
+}
+function preparedOrUndefined(source) {
+    try {
+        return prepare(source);
+    }
+    catch {
+        return undefined;
+    }
+}
+const sourceText = (item, start, end) => item.prepared.text.slice(item.prepared.codeUnits[start], item.prepared.codeUnits[end]);
+const sourceCharacters = (item, start, end) => item.prepared.codeUnits[end] - item.prepared.codeUnits[start];
+function marker(item, number) {
+    const fragment = item.fragmentIndex === undefined ? '' : ` fragment=${item.fragmentIndex}`;
+    return `\n\n[${number}] path=${JSON.stringify(item.path)} blob=${item.blobId} kind=${item.kind} lines=${item.startLine}-${item.endLine} range=[${item.primarySpan.startByte},${item.primarySpan.endByte})${fragment}\n`;
+}
+const linePrefix = (line, fragmentIndex) => fragmentIndex === undefined
+    ? `${String(line).padStart(6)}| `
+    : `${String(line).padStart(6)}[fragment ${fragmentIndex}]| `;
+function lineCompleteText(text, breaks, startLine, endLine, includeTrailingEmpty) {
+    let complete = '';
+    let cursor = 0;
+    for (const point of breaks) {
+        if (!Number.isSafeInteger(point.afterCharacter) || point.afterCharacter < cursor
+            || point.afterCharacter > text.length || !Number.isSafeInteger(point.count) || point.count <= 0)
+            return undefined;
+        complete += text.slice(cursor, point.afterCharacter) + '\n'.repeat(point.count);
+        cursor = point.afterCharacter;
+    }
+    complete += text.slice(cursor);
+    const expectedBreaks = endLine - startLine;
+    const rawBreaks = [...complete.matchAll(/\n/g)].length;
+    const actualBreaks = !includeTrailingEmpty && complete.endsWith('\n') ? rawBreaks - 1 : rawBreaks;
+    return actualBreaks === expectedBreaks ? complete : undefined;
+}
+function numberedText(text, startLine, endLine, includeTrailingEmpty, breaks, fragmentIndex) {
+    if (fragmentIndex !== undefined)
+        return linePrefix(startLine, fragmentIndex) + text;
+    const complete = lineCompleteText(text, breaks, startLine, endLine, includeTrailingEmpty);
+    if (complete === undefined)
+        return undefined;
+    let value = linePrefix(startLine);
+    let line = startLine;
+    let cursor = 0;
+    while (line < endLine) {
+        const newline = complete.indexOf('\n', cursor);
+        if (newline < 0)
+            break;
+        value += complete.slice(cursor, newline + 1);
+        cursor = newline + 1;
+        line += 1;
+        value += linePrefix(line);
+    }
+    return value + complete.slice(cursor);
+}
+function deliveredText(text, item, startLine, options, breaks, fragmentIndex) {
+    if (!options.numberLines)
+        return text;
+    return numberedText(text, startLine, fragmentIndex === undefined ? item.endLine : startLine, item.kind === 'whole', breaks, fragmentIndex) ?? invalid();
+}
+/** Complete prompts are counted with JavaScript String.length (UTF-16 code units), exactly as sent. */
+export function assembleWholeReadPrompt(prompt, contexts, items, options) {
+    let value = `${prompt.instructions}\n${prompt.schema}`;
+    for (const context of contexts)
+        value += `\n<context ref="${context.ref}">\n${context.text}\n</context>`;
+    items.forEach(({ item, primaryText, overlapText }, index) => {
+        value += marker(item, index + 1);
+        if (overlapText !== undefined) {
+            if (!item.overlapSpans[0] || item.overlapStartLine === undefined)
+                return invalid();
+            value += `<overlap context-only lines=${item.overlapStartLine}-${item.startLine - 1}>\n`
+                + deliveredText(overlapText, { ...item, endLine: item.startLine - 1 }, item.overlapStartLine, options, item.overlapReplacementBreaks ?? [])
+                + '\n</overlap>\n';
+        }
+        value += deliveredText(primaryText, item, item.startLine, options, item.primaryReplacementBreaks ?? [], item.fragmentIndex);
+    });
+    return value;
+}
+function completePrompt(prompt, contexts, items, options) {
+    return assembleWholeReadPrompt(prompt, contexts, items.map((item) => ({
+        item,
+        primaryText: sourceText(item, item.deliveredStart, item.deliveredEnd),
+        ...(item.overlapDelivered ? { overlapText: sourceText(item, ...item.overlapDelivered) } : {}),
+    })), options);
+}
+function numberedCharacterCount(text, startLine, endLine, includeTrailingEmpty, breaks, fragmentIndex) {
+    if (fragmentIndex !== undefined)
+        return text.length + linePrefix(startLine, fragmentIndex).length;
+    let previous = 0;
+    let inserted = 0;
+    for (const point of breaks) {
+        if (!Number.isSafeInteger(point.afterCharacter) || point.afterCharacter < previous
+            || point.afterCharacter > text.length || !Number.isSafeInteger(point.count) || point.count <= 0)
+            return undefined;
+        previous = point.afterCharacter;
+        inserted += point.count;
+    }
+    const rawBreaks = [...text.matchAll(/\n/g)].length + inserted;
+    const endsWithBreak = breaks.at(-1)?.afterCharacter === text.length || text.endsWith('\n');
+    const actualBreaks = !includeTrailingEmpty && endsWithBreak ? rawBreaks - 1 : rawBreaks;
+    if (actualBreaks !== endLine - startLine)
+        return undefined;
+    let count = text.length + inserted;
+    for (let line = startLine; line <= endLine; line += 1)
+        count += linePrefix(line).length;
+    return count;
+}
+function textCharacterCount(item, start, end, startLine, endLine, numberLines, breaks, fragmentIndex) {
+    if (!numberLines)
+        return sourceCharacters(item, start, end);
+    return numberedCharacterCount(sourceText(item, start, end), startLine, endLine, item.kind === 'whole', breaks, fragmentIndex);
+}
+function itemCharacterCount(item, number, numberLines) {
+    const primary = textCharacterCount(item, item.deliveredStart, item.deliveredEnd, item.startLine, item.endLine, numberLines, item.primaryReplacementBreaks ?? [], item.fragmentIndex);
+    if (primary === undefined)
+        return undefined;
+    let count = marker(item, number).length + primary;
+    if (item.overlapDelivered) {
+        if (item.overlapStartLine === undefined)
+            return invalid();
+        const overlap = textCharacterCount(item, ...item.overlapDelivered, item.overlapStartLine, item.startLine - 1, numberLines, item.overlapReplacementBreaks ?? []);
+        if (overlap === undefined)
+            return undefined;
+        count += `<overlap context-only lines=${item.overlapStartLine}-${item.startLine - 1}>\n`.length
+            + overlap + '\n</overlap>\n'.length;
+    }
+    return count;
+}
+function requiredRefs(items, limits) {
+    return new Set(items.flatMap((item) => limits.requiredContextRefs?.(publicItem(item)) ?? []));
+}
+function allowedRefs(items, limits) {
+    const refs = limits.contextRefsForItems?.(items.map(publicItem));
+    return refs ? new Set(refs) : undefined;
+}
+function fitCharacters(itemCharacters, prompt, limits, required = new Set(), allowed) {
+    const available = new Set(prompt.context.map((item) => item.ref));
+    if ([...required].some((ref) => !available.has(ref) || allowed && !allowed.has(ref)))
+        return undefined;
+    const selected = prompt.context.filter((item) => !allowed || allowed.has(item.ref));
+    let contextCharacters = selected.reduce((sum, item) => sum + `\n<context ref="${item.ref}">\n${item.text}\n</context>`.length, 0);
+    for (;;) {
+        const characters = prompt.instructions.length + 1 + prompt.schema.length + contextCharacters + itemCharacters;
+        if (characters <= limits.prompt)
+            return { characters, contextRefs: selected.map((item) => item.ref) };
+        const index = selected.findLastIndex((item) => !required.has(item.ref));
+        if (index < 0)
+            return undefined;
+        const [removed] = selected.splice(index, 1);
+        contextCharacters -= `\n<context ref="${removed.ref}">\n${removed.text}\n</context>`.length;
+    }
+}
+function fit(items, prompt, limits) {
+    let characters = 0;
+    for (const [index, item] of items.entries()) {
+        const itemCharacters = itemCharacterCount(item, index + 1, limits.numberLines);
+        if (itemCharacters === undefined)
+            return undefined;
+        characters += itemCharacters;
+    }
+    return fitCharacters(characters, prompt, limits, requiredRefs(items, limits), allowedRefs(items, limits));
+}
+function replacementBreaks(prepared, start, end) {
+    return prepared.replacementBreaks.filter((point) => point.deliveredAfter > start && point.deliveredAfter <= end)
+        .map((point) => ({ afterCharacter: prepared.codeUnits[point.deliveredAfter] - prepared.codeUnits[start],
+        count: point.count }));
+}
+function whole(prepared) {
+    const startByte = prepared.source.decoded.bom?.endByte ?? 0;
+    const endByte = prepared.source.decoded.canonicalByteLength;
+    const [deliveredStart, deliveredEnd] = prepared.deliveredSpan(startByte, endByte);
+    const primaryReplacementBreaks = replacementBreaks(prepared, deliveredStart, deliveredEnd);
+    return { prepared, entryId: prepared.source.entryId, path: prepared.source.path, blobId: prepared.source.blobId,
+        inventoryRevision: prepared.source.inventoryRevision, kind: 'whole', primarySpan: { startByte, endByte },
+        overlapSpans: [], startLine: 1, endLine: Math.max(1, prepared.source.decoded.lineCount),
+        ...(primaryReplacementBreaks.length ? { primaryReplacementBreaks } : {}),
+        deliveredStart, deliveredEnd };
+}
+function chunk(prepared, start, end) {
+    const first = prepared.lines[start];
+    const last = prepared.lines[end - 1];
+    const primarySpan = { startByte: first.startByte, endByte: last.endByte };
+    const [deliveredStart, deliveredEnd] = prepared.deliveredSpan(primarySpan.startByte, primarySpan.endByte);
+    const overlapCount = Math.min(start, 40, Math.floor((end - start) * 0.1));
+    let overlapIndex = start - overlapCount;
+    while (overlapIndex < start && !prepared.hasDeliveredBoundary(prepared.lines[overlapIndex].startByte)) {
+        overlapIndex += 1;
+    }
+    const overlapLine = overlapIndex < start ? prepared.lines[overlapIndex] : undefined;
+    const overlapSpan = overlapLine ? { startByte: overlapLine.startByte, endByte: first.startByte } : undefined;
+    const primaryReplacementBreaks = replacementBreaks(prepared, deliveredStart, deliveredEnd);
+    const overlapDelivered = overlapLine ? prepared.deliveredSpan(overlapLine.startByte, first.startByte) : undefined;
+    const overlapReplacementBreaks = overlapDelivered ? replacementBreaks(prepared, ...overlapDelivered) : [];
+    return { prepared, entryId: prepared.source.entryId, path: prepared.source.path, blobId: prepared.source.blobId,
+        inventoryRevision: prepared.source.inventoryRevision, kind: 'line_chunk', primarySpan,
+        overlapSpans: overlapSpan ? [overlapSpan] : [], startLine: first.lineNumber, endLine: last.lineNumber,
+        ...(primaryReplacementBreaks.length ? { primaryReplacementBreaks } : {}),
+        ...(overlapLine ? { overlapStartLine: overlapLine.lineNumber,
+            overlapDelivered, ...(overlapReplacementBreaks.length ? { overlapReplacementBreaks } : {}) } : {}),
+        deliveredStart, deliveredEnd };
+}
+function fragment(prepared, line, start, end, index) {
+    const primaryReplacementBreaks = replacementBreaks(prepared, start.delivered, end.delivered);
+    return { prepared, entryId: prepared.source.entryId, path: prepared.source.path, blobId: prepared.source.blobId,
+        inventoryRevision: prepared.source.inventoryRevision, kind: 'fragment',
+        primarySpan: { startByte: start.byte, endByte: end.byte }, overlapSpans: [],
+        startLine: line.lineNumber, endLine: line.lineNumber, fragmentIndex: index,
+        ...(primaryReplacementBreaks.length ? { primaryReplacementBreaks } : {}),
+        deliveredStart: start.delivered, deliveredEnd: end.delivered };
+}
+function fragmentsFor(prepared, line, prompt, limits) {
+    const points = prepared.boundaries.filter((point) => point.byte >= line.startByte && point.byte <= line.endByte
+        && !prepared.forbiddenBytes.has(point.byte));
+    if (points[0]?.byte !== line.startByte || points.at(-1)?.byte !== line.endByte)
+        return undefined;
+    const result = [];
+    let cursor = 0;
+    while (cursor < points.length - 1) {
+        let low = cursor + 1;
+        let high = points.length - 1;
+        let best = -1;
+        while (low <= high) {
+            const middle = (low + high) >>> 1;
+            if (fit([fragment(prepared, line, points[cursor], points[middle], result.length)], prompt, limits)) {
+                best = middle;
+                low = middle + 1;
+            }
+            else
+                high = middle - 1;
+        }
+        if (best < 0)
+            return undefined;
+        result.push(fragment(prepared, line, points[cursor], points[best], result.length));
+        cursor = best;
+    }
+    return result;
+}
+function itemsFor(prepared, prompt, limits) {
+    const asWhole = whole(prepared);
+    if (fit([asWhole], prompt, limits))
+        return [asWhole];
+    const lines = prepared.lines.filter((line) => line.endByte > line.startByte);
+    const validEnds = lines.map((line, index) => prepared.hasDeliveredBoundary(line.endByte) ? index + 1 : 0)
+        .filter((end) => end > 0);
+    const result = [];
+    let cursor = 0;
+    let validStart = 0;
+    while (cursor < lines.length) {
+        while (validEnds[validStart] !== undefined && validEnds[validStart] <= cursor)
+            validStart += 1;
+        let low = validStart;
+        let high = validEnds.length - 1;
+        let best = cursor;
+        while (low <= high) {
+            const middle = (low + high) >>> 1;
+            const end = validEnds[middle];
+            if (fit([chunk({ ...prepared, lines }, cursor, end)], prompt, limits)) {
+                best = end;
+                low = middle + 1;
+            }
+            else
+                high = middle - 1;
+        }
+        if (best > cursor) {
+            result.push(chunk({ ...prepared, lines }, cursor, best));
+            cursor = best;
+            continue;
+        }
+        const fragments = fragmentsFor(prepared, lines[cursor], prompt, limits);
+        if (!fragments)
+            return undefined;
+        result.push(...fragments);
+        cursor += 1;
+    }
+    return result;
+}
+const publicItem = ({ prepared: _prepared, deliveredStart: _start, deliveredEnd: _end, overlapDelivered: _overlap, ...item }) => item;
+function unit(passId, index, items, prompt, limits, parent) {
+    const fitted = fit(items, prompt, limits) ?? invalid();
+    const included = new Set(fitted.contextRefs);
+    const promptText = completePrompt(prompt, prompt.context.filter((item) => included.has(item.ref)), items, { numberLines: limits.numberLines });
+    if (promptText.length !== fitted.characters)
+        return invalid();
+    const promptHash = hash(promptText);
+    const identity = hash(JSON.stringify({ passId, parent, index, promptHash,
+        packingVersion: WHOLE_READ_PACKING_VERSION, items: items.map((item) => publicItem(item)) }));
+    return { unitId: `unit_${identity.slice(7, 31)}`, items: items.map(publicItem), contextRefs: fitted.contextRefs,
+        promptVersion: prompt.version, promptHash, promptCharacters: fitted.characters,
+        outputTokens: limits.outputEnvelope === 'per_item'
+            ? Math.min(limits.maxOutputTokens, limits.thinkingAllowanceTokens
+                + items.length * limits.outputTokensPerItem) : readUnitOutputTokens(fitted.characters) };
+}
+function result(units, setAside, failures, replacesUnitId) {
+    return { packingVersion: WHOLE_READ_PACKING_VERSION, units, setAside, failures,
+        remainingEnvelope: { unitCount: units.length,
+            promptCharacters: units.reduce((sum, item) => sum + item.promptCharacters, 0),
+            outputTokens: units.reduce((sum, item) => sum + item.outputTokens, 0) },
+        ...(replacesUnitId ? { replacesUnitId } : {}) };
+}
+function limitsFor(input) {
+    const prompt = Math.min(input.maxPromptCharacters ?? MAX_WHOLE_READ_PROMPT_CHARACTERS, MAX_WHOLE_READ_PROMPT_CHARACTERS);
+    const thinkingAllowanceTokens = input.thinkingAllowanceTokens ?? 0;
+    if (!input.passId || !input.prompt.version || !Number.isSafeInteger(prompt) || prompt <= 0
+        || typeof input.numberLines !== 'boolean'
+        || !Number.isSafeInteger(input.maxOutputTokens) || input.maxOutputTokens <= 0
+        || !Number.isSafeInteger(thinkingAllowanceTokens) || thinkingAllowanceTokens < 0
+        || !Number.isSafeInteger(input.outputTokensPerItem) || input.outputTokensPerItem <= 0
+        || (input.outputEnvelope === 'per_item'
+            && thinkingAllowanceTokens + input.outputTokensPerItem > input.maxOutputTokens))
+        return invalid();
+    return { prompt, numberLines: input.numberLines, outputEnvelope: input.outputEnvelope ?? 'read',
+        maxOutputTokens: input.maxOutputTokens, thinkingAllowanceTokens,
+        outputTokensPerItem: input.outputTokensPerItem, requiredContextRefs: input.requiredContextRefs,
+        contextRefsForItems: input.contextRefsForItems };
+}
+const outputFits = (itemCount, limits) => limits.outputEnvelope === 'read'
+    || limits.thinkingAllowanceTokens + itemCount * limits.outputTokensPerItem <= limits.maxOutputTokens;
+export function planUnits(input) {
+    const limits = limitsFor(input);
+    const setAside = [];
+    const failures = [];
+    const items = [];
+    const ids = new Set();
+    for (const source of input.entries) {
+        if (!source.entryId || ids.has(source.entryId) || !source.path
+            || !sanitizeBuildMetadata({ path: source.path }).ok || !source.blobId
+            || !Number.isSafeInteger(source.inventoryRevision))
+            invalid();
+        ids.add(source.entryId);
+        if (source.decoded.canonicalByteLength === 0) {
+            setAside.push({ entryId: source.entryId, reason: 'empty' });
+            continue;
+        }
+        const prepared = preparedOrUndefined(source);
+        let planned;
+        try {
+            planned = prepared && source.decoded.characterCount ? itemsFor(prepared, input.prompt, limits) : undefined;
+        }
+        catch {
+            planned = undefined;
+        }
+        if (planned)
+            items.push(...planned);
+        else
+            failures.push({ entryId: source.entryId, state: 'unresolved', subState: 'failed', code: 'unit_unsplittable',
+                span: { startByte: source.decoded.bom?.endByte ?? 0, endByte: source.decoded.canonicalByteLength } });
+    }
+    const groups = [];
+    let currentCharacters = 0;
+    for (const item of items) {
+        const current = groups.at(-1);
+        const itemCharacters = itemCharacterCount(item, (current?.length ?? 0) + 1, limits.numberLines) ?? invalid();
+        const appendedCharacters = currentCharacters + itemCharacters;
+        if (current && outputFits(current.length + 1, limits)
+            && fitCharacters(appendedCharacters, input.prompt, limits, requiredRefs([...current, item], limits), allowedRefs([...current, item], limits))) {
+            current.push(item);
+            currentCharacters = appendedCharacters;
+        }
+        else {
+            groups.push([item]);
+            currentCharacters = itemCharacterCount(item, 1, limits.numberLines) ?? invalid();
+        }
+    }
+    return result(groups.map((group, index) => unit(input.passId, index, group, input.prompt, limits)), setAside, failures);
+}
+function restore(item, prepared) {
+    const [deliveredStart, deliveredEnd] = prepared.deliveredSpan(item.primarySpan.startByte, item.primarySpan.endByte);
+    const overlap = item.overlapSpans[0];
+    return { ...item, path: prepared.source.path, prepared, deliveredStart, deliveredEnd,
+        ...(overlap ? { overlapDelivered: prepared.deliveredSpan(overlap.startByte, overlap.endByte) } : {}) };
+}
+function splitItem(item) {
+    const lines = item.prepared.lines.filter((line) => line.startByte >= item.primarySpan.startByte
+        && line.endByte <= item.primarySpan.endByte && line.endByte > line.startByte);
+    if (lines.length > 1) {
+        const all = item.prepared.lines.filter((line) => line.endByte > line.startByte);
+        const start = all.findIndex((line) => line.startByte === lines[0].startByte);
+        const splits = Array.from({ length: lines.length - 1 }, (_, index) => index + 1)
+            .filter((offset) => item.prepared.hasDeliveredBoundary(lines[offset].startByte));
+        const middle = splits.reduce((best, offset) => best === undefined
+            || Math.abs(offset - lines.length / 2) < Math.abs(best - lines.length / 2) ? offset : best, undefined);
+        if (middle === undefined)
+            return undefined;
+        return [chunk({ ...item.prepared, lines: all }, start, start + middle),
+            chunk({ ...item.prepared, lines: all }, start + middle, start + lines.length)];
+    }
+    const points = item.prepared.boundaries.filter((point) => point.byte >= item.primarySpan.startByte
+        && point.byte <= item.primarySpan.endByte && !item.prepared.forbiddenBytes.has(point.byte));
+    if (points.length <= 2)
+        return undefined;
+    const middle = Math.floor((points.length - 1) / 2);
+    const line = item.prepared.lines.find((candidate) => candidate.lineNumber === item.startLine);
+    if (!line)
+        return undefined;
+    const base = item.fragmentIndex ?? 0;
+    return [fragment(item.prepared, line, points[0], points[middle], base * 2),
+        fragment(item.prepared, line, points[middle], points.at(-1), base * 2 + 1)];
+}
+function fitExistingItem(item, prompt, limits) {
+    const pending = [item];
+    const fitted = [];
+    while (pending.length) {
+        const candidate = pending.shift();
+        if (fit([candidate], prompt, limits)) {
+            fitted.push(candidate);
+            continue;
+        }
+        const split = splitItem(candidate);
+        if (!split)
+            return undefined;
+        pending.unshift(...split);
+    }
+    return fitted;
+}
+/** Re-packs existing unread coverage without re-listing or re-reading inventory. */
+export function repackUnits(input) {
+    const limits = limitsFor(input);
+    if (!input.units.length)
+        return invalid();
+    const sources = new Map(input.entries.map((entry) => [entry.entryId, preparedOrUndefined(entry)]));
+    const failures = [];
+    const items = [];
+    for (const planned of input.units.flatMap((candidate) => candidate.items)) {
+        const prepared = sources.get(planned.entryId);
+        if (!prepared) {
+            failures.push({ entryId: planned.entryId, state: 'unresolved', subState: 'failed',
+                code: 'unit_unsplittable', span: planned.primarySpan });
+            continue;
+        }
+        let restored;
+        try {
+            restored = restore(planned, prepared);
+        }
+        catch {
+            failures.push({ entryId: planned.entryId, state: 'unresolved', subState: 'failed',
+                code: 'unit_unsplittable', span: planned.primarySpan });
+            continue;
+        }
+        const fitted = fitExistingItem(restored, input.prompt, limits);
+        if (fitted)
+            items.push(...fitted);
+        else
+            failures.push({ entryId: planned.entryId, state: 'unresolved', subState: 'failed',
+                code: 'unit_unsplittable', span: planned.primarySpan });
+    }
+    if (failures.length)
+        return result([], [], failures);
+    const groups = [];
+    let currentCharacters = 0;
+    for (const item of items) {
+        const current = groups.at(-1);
+        const itemCharacters = itemCharacterCount(item, (current?.length ?? 0) + 1, limits.numberLines) ?? invalid();
+        const appendedCharacters = currentCharacters + itemCharacters;
+        if (current && outputFits(current.length + 1, limits)
+            && fitCharacters(appendedCharacters, input.prompt, limits, requiredRefs([...current, item], limits), allowedRefs([...current, item], limits))) {
+            current.push(item);
+            currentCharacters = appendedCharacters;
+        }
+        else {
+            groups.push([item]);
+            currentCharacters = itemCharacterCount(item, 1, limits.numberLines) ?? invalid();
+        }
+    }
+    return result(groups.map((group, index) => unit(input.passId, index, group, input.prompt, limits, `replan:${WHOLE_READ_PACKING_VERSION}`)), [], []);
+}
+export function subdivideUnit(input) {
+    const limits = limitsFor(input);
+    const sources = new Map(input.entries.map((entry) => [entry.entryId, preparedOrUndefined(entry)]));
+    const parent = [];
+    for (const item of input.parent.items) {
+        const prepared = sources.get(item.entryId);
+        try {
+            if (!prepared)
+                throw new TypeError('whole_read_plan_invalid');
+            parent.push(restore(item, prepared));
+        }
+        catch {
+            return result([], [], [{ entryId: item.entryId, state: 'unresolved', subState: 'failed',
+                    code: 'unit_unsplittable', span: item.primarySpan }], input.parent.unitId);
+        }
+    }
+    const children = parent.length > 1 ? parent : [...(splitItem(parent[0] ?? invalid()) ?? [])];
+    if (!children.length || children.some((child) => !fit([child], input.prompt, limits))) {
+        const item = parent[0];
+        return result([], [], [{ entryId: item.entryId, state: 'unresolved', subState: 'failed',
+                code: 'unit_unsplittable', span: item.primarySpan }], input.parent.unitId);
+    }
+    return result(children.map((child, index) => unit(input.passId, index, [child], input.prompt, limits, input.parent.unitId)), [], [], input.parent.unitId);
+}