@mcoda/codali 0.1.88 → 0.1.89

package/dist/gateway/ContextPackBuilder.js

@@ -0,0 +1,317 @@
+const DEFAULT_MAX_DECISION_FACTS = 24;
+const DEFAULT_MAX_EXCERPT_CHARS = 1200;
+const DEFAULT_MAX_CONTEXT_PACK_TOKENS = 12000;
+const FRESHNESS_SCORE = {
+    fresh: 1,
+    recent: 0.75,
+    unknown: 0.4,
+    stale: 0.1,
+};
+const SOURCE_QUALITY = {
+    app_tool: 0.9,
+    docdex: 1,
+    file: 0.75,
+    github: 0.9,
+    jira: 0.9,
+    microsoft: 0.9,
+    model_observation: 0.2,
+};
+const isRecord = (value) => Boolean(value && typeof value === "object" && !Array.isArray(value));
+const positiveInteger = (value, fallback) => Number.isFinite(value) && value !== undefined && value > 0
+    ? Math.floor(value)
+    : fallback;
+const clampUnit = (value) => Number.isFinite(value) && value !== undefined
+    ? Math.max(0, Math.min(1, value))
+    : 0;
+const uniqueInOrder = (values) => {
+    const seen = new Set();
+    const output = [];
+    for (const value of values) {
+        const normalized = value.trim();
+        if (!normalized || seen.has(normalized))
+            continue;
+        seen.add(normalized);
+        output.push(normalized);
+    }
+    return output;
+};
+const estimateTokens = (value) => {
+    const text = typeof value === "string" ? value : JSON.stringify(value ?? "");
+    return Math.max(1, Math.ceil(text.length / 4));
+};
+export const estimateCodaliContextPackTokens = (pack) => estimateTokens({
+    originalQuery: pack.originalQuery,
+    decisionFacts: pack.decisionFacts.map((evidence) => ({
+        id: evidence.id,
+        claim: evidence.claim,
+        summary: evidence.summary,
+        sourceType: evidence.sourceType,
+        sourceId: evidence.sourceId,
+        sourceUri: evidence.sourceUri,
+        sourceTitle: evidence.sourceTitle,
+        confidence: evidence.confidence,
+        relevance: evidence.relevance,
+        freshness: evidence.freshness,
+        usedTool: evidence.usedTool,
+    })),
+    contradictions: pack.contradictions,
+    missingInformation: pack.missingInformation,
+    selectedExcerpts: pack.selectedExcerpts,
+    toolSummary: pack.toolSummary,
+});
+const normalizeClaimFingerprint = (claim) => claim
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, " ")
+    .trim()
+    .replace(/\s+/g, " ");
+const sourceQualityScore = (evidence) => {
+    const sourceType = evidence.sourceType.toLowerCase();
+    const direct = SOURCE_QUALITY[sourceType];
+    const base = direct ?? (sourceType.includes("docdex") ? 1 : 0.55);
+    const provenanceBoost = evidence.sourceId || evidence.sourceUri ? 0.05 : 0;
+    return Math.min(1, base + provenanceBoost);
+};
+const evidenceRankScore = (evidence) => {
+    const relevance = clampUnit(evidence.relevance);
+    const confidence = clampUnit(evidence.confidence);
+    const freshness = FRESHNESS_SCORE[evidence.freshness ?? "unknown"] ?? FRESHNESS_SCORE.unknown;
+    const sourceQuality = sourceQualityScore(evidence);
+    return (relevance * 0.42 +
+        confidence * 0.34 +
+        freshness * 0.1 +
+        sourceQuality * 0.14);
+};
+const compareEvidence = (left, right) => {
+    const scoreDiff = evidenceRankScore(right) - evidenceRankScore(left);
+    if (scoreDiff !== 0)
+        return scoreDiff;
+    const confidenceDiff = clampUnit(right.confidence) - clampUnit(left.confidence);
+    if (confidenceDiff !== 0)
+        return confidenceDiff;
+    const relevanceDiff = clampUnit(right.relevance) - clampUnit(left.relevance);
+    if (relevanceDiff !== 0)
+        return relevanceDiff;
+    return left.id.localeCompare(right.id);
+};
+const cloneEvidenceForContext = (evidence, mergedEvidenceIds = []) => {
+    const { rawExcerpt: _rawExcerpt, rawPayloadRef: _rawPayloadRef, ...rest } = evidence;
+    return {
+        ...rest,
+        metadata: {
+            ...(evidence.metadata ?? {}),
+            contextRankScore: Number(evidenceRankScore(evidence).toFixed(6)),
+            ...(mergedEvidenceIds.length > 0 ? { mergedEvidenceIds } : {}),
+        },
+    };
+};
+const selectExcerptText = (evidence, maxExcerptChars) => {
+    const raw = evidence.sourceType === "model_observation"
+        ? evidence.summary ?? evidence.claim
+        : evidence.rawExcerpt ?? evidence.summary ?? evidence.claim;
+    const normalized = raw.trim().replace(/\s+/g, " ");
+    if (normalized.length <= maxExcerptChars) {
+        return normalized;
+    }
+    return normalized.slice(0, Math.max(0, maxExcerptChars - 15)).trimEnd() + " [truncated]";
+};
+const dedupeAndRankEvidence = (evidence) => {
+    const groups = new Map();
+    for (const item of evidence) {
+        const fingerprint = normalizeClaimFingerprint(item.claim);
+        const key = fingerprint || item.id;
+        const existing = groups.get(key) ?? [];
+        existing.push(item);
+        groups.set(key, existing);
+    }
+    const ranked = [];
+    const duplicateIdsByWinner = new Map();
+    const droppedDuplicateIds = [];
+    for (const group of groups.values()) {
+        const sorted = [...group].sort(compareEvidence);
+        const winner = sorted[0];
+        if (!winner)
+            continue;
+        const duplicateIds = sorted.slice(1).map((item) => item.id);
+        if (duplicateIds.length > 0) {
+            duplicateIdsByWinner.set(winner.id, duplicateIds);
+            droppedDuplicateIds.push(...duplicateIds);
+        }
+        ranked.push(winner);
+    }
+    return {
+        ranked: ranked.sort(compareEvidence),
+        duplicateIdsByWinner,
+        droppedDuplicateIds,
+    };
+};
+const summarizeTools = (calls) => {
+    const byTool = new Map();
+    for (const call of calls) {
+        const statuses = byTool.get(call.tool) ?? {};
+        statuses[call.status] = (statuses[call.status] ?? 0) + 1;
+        byTool.set(call.tool, statuses);
+    }
+    return [...byTool.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([tool, statuses]) => ({
+        tool,
+        calls: Object.values(statuses).reduce((sum, value) => sum + value, 0),
+        statuses,
+    }));
+};
+const readRequestFromTrace = (trace) => {
+    const request = trace?.run.request;
+    if (!isRecord(request) || typeof request.query !== "string") {
+        return undefined;
+    }
+    return request;
+};
+const readVerificationFromTrace = (trace) => {
+    const metadata = trace?.run.metadata;
+    if (!isRecord(metadata) || !isRecord(metadata.verification)) {
+        return undefined;
+    }
+    return metadata.verification;
+};
+const normalizeContradictions = (contradictions) => {
+    const seen = new Set();
+    const output = [];
+    for (const contradiction of contradictions) {
+        const summary = contradiction.summary.trim();
+        if (!summary)
+            continue;
+        const evidenceIds = uniqueInOrder(contradiction.evidenceIds ?? []);
+        const key = `${summary}:${evidenceIds.join(",")}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        output.push({ summary, evidenceIds });
+    }
+    return output;
+};
+export const buildCodaliContextPack = (input) => {
+    const traceRequest = readRequestFromTrace(input.trace);
+    const request = input.request ?? traceRequest;
+    const verification = input.verification ?? readVerificationFromTrace(input.trace);
+    const maxContextPackTokens = positiveInteger(input.maxContextPackTokens ?? request?.policy?.maxContextPackTokens, DEFAULT_MAX_CONTEXT_PACK_TOKENS);
+    const maxDecisionFacts = positiveInteger(input.maxDecisionFacts ?? request?.policy?.maxEvidenceItems, DEFAULT_MAX_DECISION_FACTS);
+    const maxExcerptChars = positiveInteger(input.maxExcerptChars, DEFAULT_MAX_EXCERPT_CHARS);
+    const evidence = input.evidence ?? input.trace?.evidence ?? [];
+    const toolCalls = input.toolCalls ?? input.trace?.toolCalls ?? [];
+    const originalQuery = input.originalQuery ??
+        request?.query ??
+        (typeof input.trace?.run.request === "object" &&
+            input.trace?.run.request &&
+            "query" in input.trace.run.request &&
+            typeof input.trace.run.request.query === "string"
+            ? input.trace.run.request.query
+            : "");
+    const warnings = [];
+    const { ranked, duplicateIdsByWinner, droppedDuplicateIds } = dedupeAndRankEvidence(evidence);
+    const missingInformation = uniqueInOrder(verification?.missingInformation ?? []);
+    const contradictions = normalizeContradictions(verification?.contradictions ?? []);
+    const toolSummary = summarizeTools(toolCalls);
+    const decisionFacts = [];
+    const selectedExcerpts = [];
+    const selectedEvidenceIds = [];
+    const droppedEvidenceIds = new Set(droppedDuplicateIds);
+    let contextPack = {
+        id: `context-pack-${input.runId}`,
+        runId: input.runId,
+        originalQuery,
+        decisionFacts,
+        contradictions,
+        missingInformation,
+        selectedExcerpts,
+        toolSummary,
+        tokenEstimate: 1,
+        metadata: {
+            maxContextPackTokens,
+            sourceEvidenceCount: evidence.length,
+            deduplicatedEvidenceCount: ranked.length,
+            duplicateEvidenceIds: droppedDuplicateIds,
+        },
+    };
+    let tokenEstimate = estimateCodaliContextPackTokens(contextPack);
+    for (const evidenceItem of ranked) {
+        if (decisionFacts.length >= maxDecisionFacts) {
+            droppedEvidenceIds.add(evidenceItem.id);
+            continue;
+        }
+        const fact = cloneEvidenceForContext(evidenceItem, duplicateIdsByWinner.get(evidenceItem.id) ?? []);
+        const excerptText = selectExcerptText(evidenceItem, maxExcerptChars);
+        const excerpt = {
+            evidenceId: evidenceItem.id,
+            text: excerptText,
+        };
+        const candidatePack = {
+            ...contextPack,
+            decisionFacts: [...decisionFacts, fact],
+            selectedExcerpts: [...selectedExcerpts, excerpt],
+            tokenEstimate,
+        };
+        const candidateEstimate = estimateCodaliContextPackTokens(candidatePack);
+        if (candidateEstimate > maxContextPackTokens && decisionFacts.length > 0) {
+            droppedEvidenceIds.add(evidenceItem.id);
+            warnings.push(`context_pack_budget_dropped:${evidenceItem.id}`);
+            continue;
+        }
+        decisionFacts.push(fact);
+        selectedExcerpts.push(excerpt);
+        selectedEvidenceIds.push(evidenceItem.id);
+        tokenEstimate = Math.min(candidateEstimate, maxContextPackTokens);
+        contextPack = {
+            ...candidatePack,
+            tokenEstimate,
+        };
+    }
+    for (const evidenceItem of ranked) {
+        if (!selectedEvidenceIds.includes(evidenceItem.id)) {
+            droppedEvidenceIds.add(evidenceItem.id);
+        }
+    }
+    contextPack = {
+        ...contextPack,
+        decisionFacts,
+        selectedExcerpts,
+        tokenEstimate: Math.min(estimateCodaliContextPackTokens(contextPack), maxContextPackTokens),
+        metadata: {
+            ...(contextPack.metadata ?? {}),
+            selectedEvidenceIds,
+            droppedEvidenceIds: [...droppedEvidenceIds].sort(),
+            warningCount: warnings.length,
+        },
+    };
+    return {
+        contextPack,
+        selectedEvidenceIds,
+        droppedEvidenceIds: [...droppedEvidenceIds].sort(),
+        warnings,
+    };
+};
+export class CodaliContextPackBuilder {
+    constructor(options) {
+        this.options = options;
+    }
+    async buildAndPersist(input) {
+        const trace = await this.options.store.readRunTrace(input.runId);
+        if (!trace) {
+            throw new Error(`GATEWAY_RUN_NOT_FOUND: ${input.runId}`);
+        }
+        const result = buildCodaliContextPack({
+            runId: input.runId,
+            request: input.request,
+            trace,
+            verification: input.verification,
+            maxContextPackTokens: input.maxContextPackTokens,
+            maxDecisionFacts: input.maxDecisionFacts ?? this.options.maxDecisionFacts,
+            maxExcerptChars: input.maxExcerptChars ?? this.options.maxExcerptChars,
+        });
+        const saved = await this.options.store.saveContextPack(input.runId, result.contextPack);
+        return {
+            ...result,
+            contextPack: saved,
+        };
+    }
+}
+export const createCodaliContextPackBuilder = (options) => new CodaliContextPackBuilder(options);

package/dist/gateway/EvidenceNormalizer.d.ts

@@ -0,0 +1,42 @@
+import type { CodaliEvidenceItem } from "./CodaliGatewayTypes.js";
+export interface CodaliEvidenceNormalizerToolCall {
+    tool: string;
+    status?: "success" | "failed" | "blocked" | string;
+    args?: unknown;
+    result?: unknown;
+    errorCode?: string;
+    errorMessage?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliEvidenceNormalizerInput {
+    runId: string;
+    taskId?: string;
+    stageId?: string;
+    originalQuery?: string;
+    workerOutput?: unknown;
+    evidence?: unknown[];
+    toolCalls?: CodaliEvidenceNormalizerToolCall[];
+    requireTenantScope?: boolean;
+    defaultTenantScoped?: boolean;
+    defaultSourceType?: string;
+    defaultTool?: string;
+    maxEvidenceItems?: number;
+}
+export interface CodaliEvidenceRejectedItem {
+    reason: string;
+    claim?: string;
+    sourceType?: string;
+    sourceId?: string;
+    sourceUri?: string;
+    usedTool?: string;
+    value?: unknown;
+}
+export interface CodaliEvidenceNormalizerResult {
+    evidence: CodaliEvidenceItem[];
+    rejected: CodaliEvidenceRejectedItem[];
+    warnings: string[];
+    duplicateCount: number;
+}
+export declare const normalizeCodaliEvidence: (input: CodaliEvidenceNormalizerInput) => CodaliEvidenceNormalizerResult;
+export declare const normalizeGatewayEvidence: (input: CodaliEvidenceNormalizerInput) => CodaliEvidenceNormalizerResult;
+//# sourceMappingURL=EvidenceNormalizer.d.ts.map

package/dist/gateway/EvidenceNormalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvidenceNormalizer.d.ts","sourceRoot":"","sources":["../../src/gateway/EvidenceNormalizer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAEnB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;IACnD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,6BAA6B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,gCAAgC,EAAE,CAAC;IAC/C,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,QAAQ,EAAE,kBAAkB,EAAE,CAAC;IAC/B,QAAQ,EAAE,0BAA0B,EAAE,CAAC;IACvC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AA0eD,eAAO,MAAM,uBAAuB,GAClC,OAAO,6BAA6B,KACnC,8BA+HF,CAAC;AAEF,eAAO,MAAM,wBAAwB,UAlI5B,6BAA6B,KACnC,8BAiI4D,CAAC"}