@withtruing/companion 0.13.2 → 0.14.0

package/dist/whole-read/pages.js

@@ -0,0 +1,429 @@
+import { createHash } from 'node:crypto';
+import { canonicalJson } from './identity.js';
+import { sanitizeBuildMetadata } from './sanitize.js';
+import { WHOLE_READ_PAGE_PROTOCOL, } from './types.js';
+export const MAX_METADATA_PAGE_BYTES = 512 * 1024;
+export const MAX_TRANSPORT_RECORD_BYTES = 16 * 1024;
+const COLLECTION_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const SEMANTIC_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,95}$/;
+const GENERATED_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/;
+const RECORD_KIND = /^[a-z][a-z0-9_-]{0,63}$/;
+const DIGEST = /^sha256:[0-9a-f]{64}$/;
+const BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+const COLLECTION_KINDS = new Set(['inventory', 'notes', 'groups', 'claims', 'progress']);
+const CHILD_FIELDS = {
+    group: ['memberships', 'files'],
+    note: ['facts', 'exposes', 'uses'],
+    claim: ['citations'],
+};
+const CHILD_KINDS = {
+    memberships: 'membership', files: 'membership', facts: 'fact',
+    exposes: 'expose', uses: 'use', citations: 'citation',
+};
+function fail(code) { throw new Error(code); }
+const byteCount = (value) => Buffer.byteLength(value, 'utf8');
+const isObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
+const isJsonArray = (value) => Array.isArray(value);
+function validCollectionId(value) {
+    return typeof value === 'string' && COLLECTION_ID.test(value);
+}
+function validSemanticId(value, generated = false) {
+    return typeof value === 'string' && (generated ? GENERATED_ID : SEMANTIC_ID).test(value);
+}
+function canonical(value) {
+    try {
+        return canonicalJson(value);
+    }
+    catch {
+        return fail('collection_invalid');
+    }
+}
+function framedDigest(version, frames) {
+    const hash = createHash('sha256');
+    for (const frame of [version, ...frames]) {
+        const bytes = Buffer.from(canonical(frame), 'utf8');
+        hash.update(String(bytes.byteLength));
+        hash.update(':');
+        hash.update(bytes);
+        hash.update(';');
+    }
+    return `sha256:${hash.digest('hex')}`;
+}
+function assertLogicalRecord(record) {
+    if (!validCollectionId(record.collectionId) || !COLLECTION_KINDS.has(record.collectionKind)
+        || !validSemanticId(record.semanticId) || !RECORD_KIND.test(record.recordKind)
+        || !isObject(record.payload))
+        fail('collection_invalid');
+    if (!sanitizeBuildMetadata(record.payload).ok)
+        fail('unsafe_metadata');
+    canonical(record.payload);
+}
+function childId(parentId, field, order) {
+    const id = `${parentId}:${field}:${order}`;
+    if (!validSemanticId(id, true))
+        fail('collection_invalid');
+    return id;
+}
+/** Normalize unbounded repeated fields before considering fragmentation. */
+export function normalizeLogicalRecord(record) {
+    assertLogicalRecord(record);
+    const childFields = CHILD_FIELDS[record.recordKind] ?? [];
+    const payload = { ...record.payload };
+    const childSets = [];
+    const children = [];
+    for (const field of childFields) {
+        if (!(field in payload))
+            continue;
+        const entries = payload[field];
+        if (!isJsonArray(entries))
+            fail('collection_invalid');
+        delete payload[field];
+        childSets.push({ field, count: entries.length });
+        for (let order = 0; order < entries.length; order += 1) {
+            children.push({
+                type: 'child',
+                semanticId: childId(record.semanticId, field, order),
+                parentId: record.semanticId,
+                childKind: CHILD_KINDS[field],
+                field,
+                order,
+                payload: entries[order],
+            });
+        }
+    }
+    const parent = {
+        type: 'logical', semanticId: record.semanticId, recordKind: record.recordKind,
+        payload, childSets,
+    };
+    return [parent, ...children];
+}
+function directSemanticId(record) {
+    return record.type === 'fragment' ? record.parentId : record.semanticId;
+}
+function fragmentsFor(record) {
+    const bytes = Buffer.from(canonical(record), 'utf8');
+    let chunkBytes = 12 * 1024;
+    while (chunkBytes > 0) {
+        const chunks = [];
+        for (let start = 0; start < bytes.byteLength;) {
+            let end = Math.min(bytes.byteLength, start + chunkBytes);
+            while (end < bytes.byteLength && end > start && (bytes[end] & 0xc0) === 0x80)
+                end -= 1;
+            if (end === start)
+                fail('record_too_large');
+            chunks.push(bytes.subarray(start, end));
+            start = end;
+        }
+        const fragments = chunks.map((chunk, fragmentIndex) => ({
+            type: 'fragment',
+            parentId: directSemanticId(record),
+            fragmentIndex,
+            fragmentCount: chunks.length,
+            payloadEncoding: 'base64',
+            payload: chunk.toString('base64'),
+        }));
+        const largest = Math.max(...fragments.map(transportRecordByteCount));
+        if (largest <= MAX_TRANSPORT_RECORD_BYTES)
+            return fragments;
+        chunkBytes -= Math.max(1, Math.ceil((largest - MAX_TRANSPORT_RECORD_BYTES) * 3 / 4) + 8);
+    }
+    return fail('record_too_large');
+}
+function boundedRecords(records) {
+    const result = [];
+    const semanticIds = new Set();
+    for (const record of records) {
+        if (record.type === 'fragment')
+            fail('collection_invalid');
+        const id = directSemanticId(record);
+        if (semanticIds.has(id))
+            fail('collection_invalid');
+        semanticIds.add(id);
+        if (transportRecordByteCount(record) <= MAX_TRANSPORT_RECORD_BYTES)
+            result.push(record);
+        else
+            result.push(...fragmentsFor(record));
+    }
+    return result;
+}
+export function transportRecordByteCount(record) {
+    return byteCount(canonical(record));
+}
+function pageDigestFields(page) {
+    return [{
+            collectionId: page.collectionId,
+            kind: page.kind,
+            pageIndex: page.pageIndex,
+            protocolVersion: page.protocolVersion,
+            recordCount: page.recordCount,
+        }, ...page.records];
+}
+export function metadataPageDigest(page) {
+    return framedDigest('whole-read-metadata-page-digest@1', pageDigestFields(page));
+}
+export function metadataPageByteCount(page) {
+    return byteCount(canonical(page));
+}
+function pageEnvelopeBytes(identity, pageIndex, recordCount) {
+    return metadataPageByteCount({
+        protocolVersion: WHOLE_READ_PAGE_PROTOCOL,
+        collectionId: identity.collectionId,
+        kind: identity.kind,
+        pageIndex,
+        recordCount,
+        records: [],
+        digest: `sha256:${'0'.repeat(64)}`,
+    });
+}
+function makePage(identity, pageIndex, records) {
+    const fields = {
+        protocolVersion: WHOLE_READ_PAGE_PROTOCOL,
+        collectionId: identity.collectionId,
+        kind: identity.kind,
+        pageIndex,
+        recordCount: records.length,
+        records,
+    };
+    return { ...fields, digest: metadataPageDigest(fields) };
+}
+/** Encode records in declared order. An empty iterable intentionally produces zero pages. */
+export function* encodeCollection(records) {
+    let identity;
+    const normalized = [];
+    const logicalIds = new Set();
+    for (const record of records) {
+        assertLogicalRecord(record);
+        identity ??= { collectionId: record.collectionId, kind: record.collectionKind };
+        if (record.collectionId !== identity.collectionId || record.collectionKind !== identity.kind
+            || logicalIds.has(record.semanticId))
+            fail('collection_invalid');
+        logicalIds.add(record.semanticId);
+        normalized.push(...normalizeLogicalRecord(record));
+    }
+    if (!identity)
+        return;
+    const transport = boundedRecords(normalized);
+    let pageIndex = 0;
+    let current = [];
+    let currentRecordBytes = 0;
+    for (const record of transport) {
+        const recordBytes = transportRecordByteCount(record);
+        const nextCount = current.length + 1;
+        const nextBytes = pageEnvelopeBytes(identity, pageIndex, nextCount)
+            + currentRecordBytes + recordBytes + (nextCount > 1 ? nextCount - 1 : 0);
+        if (nextBytes > MAX_METADATA_PAGE_BYTES && current.length > 0) {
+            yield makePage(identity, pageIndex, current);
+            pageIndex += 1;
+            current = [];
+            currentRecordBytes = 0;
+        }
+        const emptyPageBytes = pageEnvelopeBytes(identity, pageIndex, 1) + recordBytes;
+        if (emptyPageBytes > MAX_METADATA_PAGE_BYTES)
+            fail('page_too_large');
+        current.push(record);
+        currentRecordBytes += recordBytes;
+    }
+    if (current.length > 0)
+        yield makePage(identity, pageIndex, current);
+}
+function assertTransportRecord(record) {
+    if (!isObject(record) || typeof record.type !== 'string')
+        fail('collection_invalid');
+    if (record.type === 'logical') {
+        if (Object.keys(record).sort().join(',') !== 'childSets,payload,recordKind,semanticId,type'
+            || !validSemanticId(record.semanticId, true) || !RECORD_KIND.test(String(record.recordKind))
+            || !isObject(record.payload) || !Array.isArray(record.childSets))
+            fail('collection_invalid');
+        const fields = new Set();
+        for (const childSet of record.childSets) {
+            if (!isObject(childSet) || Object.keys(childSet).sort().join(',') !== 'count,field'
+                || typeof childSet.field !== 'string'
+                || !(childSet.field in CHILD_KINDS) || fields.has(childSet.field)
+                || !Number.isSafeInteger(childSet.count) || Number(childSet.count) < 0)
+                fail('collection_invalid');
+            fields.add(childSet.field);
+        }
+    }
+    else if (record.type === 'child') {
+        if (Object.keys(record).sort().join(',') !== 'childKind,field,order,parentId,payload,semanticId,type'
+            || !validSemanticId(record.semanticId, true) || !validSemanticId(record.parentId)
+            || typeof record.field !== 'string' || CHILD_KINDS[record.field] !== record.childKind
+            || !Number.isSafeInteger(record.order) || Number(record.order) < 0
+            || !('payload' in record))
+            fail('collection_invalid');
+    }
+    else if (record.type === 'fragment') {
+        if (Object.keys(record).sort().join(',') !== 'fragmentCount,fragmentIndex,parentId,payload,payloadEncoding,type'
+            || !validSemanticId(record.parentId, true) || !Number.isSafeInteger(record.fragmentIndex)
+            || Number(record.fragmentIndex) < 0 || !Number.isSafeInteger(record.fragmentCount)
+            || Number(record.fragmentCount) < 1 || Number(record.fragmentIndex) >= Number(record.fragmentCount)
+            || record.payloadEncoding !== 'base64' || typeof record.payload !== 'string'
+            || !BASE64.test(record.payload))
+            fail('collection_invalid');
+        let decoded;
+        try {
+            decoded = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.from(record.payload, 'base64'));
+        }
+        catch {
+            return fail('collection_invalid');
+        }
+        if (!sanitizeBuildMetadata(decoded).ok)
+            fail('unsafe_metadata');
+    }
+    else
+        fail('collection_invalid');
+    if (!sanitizeBuildMetadata(record).ok)
+        fail('unsafe_metadata');
+    canonical(record);
+}
+export function validateMetadataPage(page) {
+    if (!isObject(page) || page.protocolVersion !== WHOLE_READ_PAGE_PROTOCOL
+        || !validCollectionId(page.collectionId) || !COLLECTION_KINDS.has(page.kind)
+        || !Number.isSafeInteger(page.pageIndex) || page.pageIndex < 0
+        || !Number.isSafeInteger(page.recordCount) || page.recordCount < 1
+        || !Array.isArray(page.records) || page.recordCount !== page.records.length
+        || !DIGEST.test(page.digest))
+        fail('collection_invalid');
+    if (Object.keys(page).sort().join(',') !== 'collectionId,digest,kind,pageIndex,protocolVersion,recordCount,records'
+        || !sanitizeBuildMetadata(page).ok)
+        fail('unsafe_metadata');
+    for (const record of page.records) {
+        assertTransportRecord(record);
+        if (transportRecordByteCount(record) > MAX_TRANSPORT_RECORD_BYTES)
+            fail('record_too_large');
+    }
+    if (metadataPageByteCount(page) > MAX_METADATA_PAGE_BYTES)
+        fail('page_too_large');
+    const { digest: _digest, ...fields } = page;
+    if (metadataPageDigest(fields) !== page.digest)
+        fail('collection_digest_mismatch');
+}
+function orderedPages(pages) {
+    const ordered = [...pages].sort((left, right) => left.pageIndex - right.pageIndex);
+    if (ordered.length === 0)
+        return ordered;
+    const identity = { collectionId: ordered[0].collectionId, kind: ordered[0].kind };
+    for (let index = 0; index < ordered.length; index += 1) {
+        const page = ordered[index];
+        if (page.pageIndex !== index)
+            fail('collection_incomplete');
+        validateMetadataPage(page);
+        if (page.collectionId !== identity.collectionId || page.kind !== identity.kind)
+            fail('collection_invalid');
+    }
+    return ordered;
+}
+export function collectionDigest(pages, emptyIdentity) {
+    const ordered = orderedPages(pages);
+    const identity = ordered.length > 0
+        ? { collectionId: ordered[0].collectionId, kind: ordered[0].kind }
+        : emptyIdentity;
+    if (!identity || !validCollectionId(identity.collectionId) || !COLLECTION_KINDS.has(identity.kind)) {
+        return fail('collection_invalid');
+    }
+    return framedDigest('whole-read-collection-digest@1', [
+        { ...identity, pageCount: ordered.length },
+        ...ordered.map((page) => ({
+            digest: page.digest, pageIndex: page.pageIndex, recordCount: page.recordCount,
+        })),
+    ]);
+}
+function decodedFragments(records) {
+    const result = [];
+    const fragments = new Map();
+    const emitted = new Set();
+    for (const record of records) {
+        if (record.type !== 'fragment') {
+            result.push(record);
+            continue;
+        }
+        const group = fragments.get(record.parentId) ?? [];
+        group.push(record);
+        fragments.set(record.parentId, group);
+        if (emitted.has(record.parentId))
+            continue;
+        emitted.add(record.parentId);
+        result.push({ type: 'logical', semanticId: record.parentId, recordKind: 'fragment_placeholder', payload: {}, childSets: [] });
+    }
+    for (let index = 0; index < result.length; index += 1) {
+        const placeholder = result[index];
+        if (placeholder.type !== 'logical' || placeholder.recordKind !== 'fragment_placeholder')
+            continue;
+        const parentId = placeholder.semanticId;
+        const group = fragments.get(parentId) ?? [];
+        const expected = group[0]?.fragmentCount;
+        if (!expected || group.length !== expected)
+            fail('collection_incomplete');
+        group.sort((left, right) => left.fragmentIndex - right.fragmentIndex);
+        if (group.some((fragment, fragmentIndex) => fragment.fragmentIndex !== fragmentIndex
+            || fragment.fragmentCount !== expected))
+            fail('collection_incomplete');
+        let parsed;
+        try {
+            parsed = JSON.parse(Buffer.concat(group.map((fragment) => Buffer.from(fragment.payload, 'base64'))).toString('utf8'));
+        }
+        catch {
+            fail('collection_invalid');
+        }
+        assertTransportRecord(parsed);
+        if (parsed.type === 'fragment' || directSemanticId(parsed) !== parentId)
+            fail('collection_invalid');
+        result[index] = parsed;
+    }
+    return result;
+}
+export function decodeCollection(pages, expectedDigest) {
+    const ordered = orderedPages(pages);
+    if (ordered.length === 0)
+        return [];
+    if (expectedDigest !== undefined && collectionDigest(ordered) !== expectedDigest) {
+        fail('collection_digest_mismatch');
+    }
+    const records = decodedFragments(ordered.flatMap((page) => page.records));
+    const semanticIds = new Set();
+    const parents = [];
+    const children = new Map();
+    for (const record of records) {
+        if (semanticIds.has(record.semanticId))
+            fail('collection_invalid');
+        semanticIds.add(record.semanticId);
+        if (record.type === 'logical')
+            parents.push(record);
+        else {
+            const entries = children.get(record.parentId) ?? [];
+            entries.push(record);
+            children.set(record.parentId, entries);
+        }
+    }
+    const consumedChildren = new Set();
+    const identity = ordered[0];
+    const logical = parents.map((parent) => {
+        const payload = { ...parent.payload };
+        const parentChildren = children.get(parent.semanticId) ?? [];
+        for (const childSet of parent.childSets) {
+            if (childSet.field in payload)
+                fail('collection_invalid');
+            const matching = parentChildren.filter((child) => child.field === childSet.field)
+                .sort((left, right) => left.order - right.order);
+            if (matching.length !== childSet.count
+                || matching.some((child, order) => child.order !== order
+                    || child.semanticId !== childId(parent.semanticId, childSet.field, order))) {
+                fail('collection_invalid');
+            }
+            payload[childSet.field] = matching.map((child) => child.payload);
+            for (const child of matching)
+                consumedChildren.add(child.semanticId);
+        }
+        return {
+            collectionId: identity.collectionId,
+            collectionKind: identity.kind,
+            semanticId: parent.semanticId,
+            recordKind: parent.recordKind,
+            payload,
+        };
+    });
+    if ([...children.values()].flat().some((child) => !consumedChildren.has(child.semanticId))) {
+        fail('collection_invalid');
+    }
+    return logical;
+}

package/dist/whole-read/phase-policy.js

@@ -0,0 +1,54 @@
+/** Consent-frozen settings and reservation envelopes for every whole-read model phase. */
+export const WHOLE_READ_PHASE_POLICY = {
+    read: { model: 'claude-haiku-5-5', fallbackModel: 'claude-sonnet-5-5', effort: 'low',
+        thinking: { 'claude-haiku-5-5': 'disabled', 'claude-sonnet-5-5': 'between_tools' },
+        maxOutputTokens: 16_000, thinkingAllowanceTokens: 0, perItemOutputTokens: 512 },
+    // 16,384 - 2,048 thinking leaves 14,336 answer tokens. At 20 per card (a short number plus
+    // punctuation and amortized group-label overhead), one unit can return 716 cards; 1,200 fit in two.
+    grouping: { model: 'claude-opus-5-5', fallbackModel: 'claude-sonnet-5-5', effort: 'low',
+        thinking: { 'claude-opus-5-5': 'adaptive', 'claude-sonnet-5-5': 'adaptive' },
+        maxOutputTokens: 16_384, thinkingAllowanceTokens: 2_048, perItemOutputTokens: 20 },
+    // 24,576 leaves medium effort 8,192 thinking tokens once, then fits main's 16 files.
+    drafting_leaf: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'medium',
+        thinking: { 'claude-sonnet-5-5': 'adaptive', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 24_576, thinkingAllowanceTokens: 8_192, perItemOutputTokens: 1_024 },
+    drafting_merge: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'medium',
+        thinking: { 'claude-sonnet-5-5': 'adaptive', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 24_576, thinkingAllowanceTokens: 8_192, perItemOutputTokens: 1_024 },
+    support_check: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'low',
+        thinking: { 'claude-sonnet-5-5': 'between_tools', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 4_096, thinkingAllowanceTokens: 0, perItemOutputTokens: 512 },
+    // The 10,240 cap preserves main's 16-candidate view after Opus's one-time allowance.
+    relationships: { model: 'claude-opus-5-5', fallbackModel: 'claude-sonnet-5-5', effort: 'low',
+        thinking: { 'claude-opus-5-5': 'adaptive', 'claude-sonnet-5-5': 'adaptive' },
+        maxOutputTokens: 10_240, thinkingAllowanceTokens: 2_048, perItemOutputTokens: 512 },
+    // The 34,048 cap preserves main's 200-item reconciliation units plus one Opus allowance.
+    reconciliation: { model: 'claude-opus-5-5', fallbackModel: 'claude-sonnet-5-5', effort: 'low',
+        thinking: { 'claude-opus-5-5': 'adaptive', 'claude-sonnet-5-5': 'adaptive' },
+        maxOutputTokens: 34_048, thinkingAllowanceTokens: 2_048, perItemOutputTokens: 160 },
+    // PRD answer limits stay 4k/16k/8k; each request adds medium adaptive thinking once per unit.
+    prd_lanes: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'medium',
+        thinking: { 'claude-sonnet-5-5': 'adaptive', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 12_288, thinkingAllowanceTokens: 8_192, perItemOutputTokens: 4_096 },
+    prd_draft: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'medium',
+        thinking: { 'claude-sonnet-5-5': 'adaptive', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 24_576, thinkingAllowanceTokens: 8_192, perItemOutputTokens: 16_384 },
+    prd_check: { model: 'claude-sonnet-5-5', fallbackModel: 'claude-opus-5-5', effort: 'medium',
+        thinking: { 'claude-sonnet-5-5': 'adaptive', 'claude-opus-5-5': 'adaptive' },
+        maxOutputTokens: 16_384, thinkingAllowanceTokens: 8_192, perItemOutputTokens: 8_192 },
+};
+export function phaseItemsPerUnit(policy) {
+    return Math.floor((policy.maxOutputTokens - policy.thinkingAllowanceTokens) / policy.perItemOutputTokens);
+}
+export function phaseUnitOutputTokens(policy, itemCount) {
+    return Math.min(policy.maxOutputTokens, policy.thinkingAllowanceTokens + itemCount * policy.perItemOutputTokens);
+}
+export function anthropicThinking(phase, model) {
+    const thinking = WHOLE_READ_PHASE_POLICY[phase].thinking[model];
+    if (!thinking)
+        throw new TypeError('whole_read_model_policy_invalid');
+    return { type: thinking };
+}
+export function prdPolicyPhase(phase) {
+    return `prd_${phase}`;
+}