@clue-ai/cli 0.0.3 → 0.0.5

package/src/semantic-ci.mjs

@@ -2,6 +2,7 @@ import { createHash, createHmac } from "node:crypto";
 import {
   validateSemanticCiRequest,
   validateSemanticSnapshotRequest,
+  validateSemanticSnapshotResponse,
 } from "./contracts.mjs";
 import { analyzeFastApiRoutes } from "./fastapi-analyzer.mjs";
 import { listAllowedPythonFiles } from "./path-policy.mjs";
@@ -73,10 +74,43 @@ const semanticSnapshotHashScope = (request) =>
     "clue_tools_semantic_snapshot",
     request.project_key,
     request.repository.repository_id,
-    request.repository.merge_commit,
-    request.repository.workflow_run_id ?? "no_workflow_run",
+    request.service.service_key,
   ].join(":");
 
+const canonicalJson = (value) => {
+  if (Array.isArray(value)) {
+    return `[${value.map((entry) => canonicalJson(entry)).join(",")}]`;
+  }
+  if (value && typeof value === "object") {
+    return `{${Object.keys(value)
+      .sort()
+      .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
+      .join(",")}}`;
+  }
+  return JSON.stringify(value);
+};
+
+const routeInputHash = (route) =>
+  sha256(
+    canonicalJson({
+      operation_source_key: route.operation_source_key,
+      method: route.method,
+      path_template: route.path_template,
+      source: route.source,
+      code_structure: route.ai_context?.code_structure ?? null,
+    }),
+  );
+
+const routeSemanticHash = (route) =>
+  sha256(
+    canonicalJson({
+      semantics: route.semantics,
+      layer_evidence: route.layer_evidence,
+      operation_effects: route.operation_effects ?? [],
+      unresolved_operation_effects: route.unresolved_operation_effects ?? [],
+    }),
+  );
+
 const opaqueRouteHash = (route, purpose, hashScope) =>
   hmacSha256(
     hashScope,
@@ -165,6 +199,134 @@ const buildAiPrompt = (routes) =>
     })),
   });
 
+const buildAiReuseDecisionPrompt = ({ route, previousRoute, currentHash }) =>
+  JSON.stringify({
+    task: "Decide whether previous privacy-safe route semantics still apply after a route source change. Return JSON only.",
+    rules: [
+      "Only decide for the provided operation_source_key.",
+      "Return decision as reuse_previous, regenerate, or needs_review.",
+      "Choose reuse_previous only when the customer-visible route meaning, operation effects, and value interpretation remain the same.",
+      "Choose regenerate when the route appears to create, update, delete, read, send, schedule, call, or validate a different business object or effect.",
+      "Choose needs_review when evidence is insufficient or conflicting.",
+      "Do not include raw source code, raw SQL, file paths, function names, class names, prompts, completions, secrets, ids, or hashes except the provided hashes.",
+    ],
+    output_shape: {
+      decisions: [
+        {
+          operation_source_key: "route.POST./reports",
+          decision: "reuse_previous",
+          confidence: 0.82,
+          reason:
+            "The source changed but the privacy-safe evidence still supports the previous reports.create meaning.",
+        },
+      ],
+    },
+    route: {
+      operation_source_key: route.operation_source_key,
+      method: route.method,
+      path_template: route.path_template,
+      current_route_input_hash: currentHash,
+      previous_route_input_hash: previousRoute.route_input_hash ?? null,
+      previous_semantics: previousRoute.semantics,
+      previous_operation_effects: previousRoute.operation_effects ?? [],
+      current_evidence_summaries: route.evidence_summaries,
+    },
+  });
+
+const normalizeReuseDecision = ({ parsed, operationSourceKey }) => {
+  const decision = safeArray(parsed?.decisions).find(
+    (entry) => entry?.operation_source_key === operationSourceKey,
+  );
+  if (!decision || typeof decision !== "object") {
+    return {
+      decision: "needs_review",
+      confidence: 0,
+      reason: "AI reuse decision omitted this changed route.",
+    };
+  }
+  const normalizedDecision = ["reuse_previous", "regenerate", "needs_review"].includes(
+    decision.decision,
+  )
+    ? decision.decision
+    : "needs_review";
+  return {
+    decision: normalizedDecision,
+    confidence: Number.isFinite(Number(decision.confidence))
+      ? Math.max(0, Math.min(1, Number(decision.confidence)))
+      : 0,
+    reason:
+      typeof decision.reason === "string" && decision.reason.trim()
+        ? decision.reason.trim()
+        : "AI reuse decision did not include a reason.",
+  };
+};
+
+const callAiReuseDecisionProvider = async ({
+  request,
+  apiKey,
+  route,
+  previousRoute,
+  currentHash,
+}) => {
+  const response = await fetch(
+    `${request.ai_provider_base_url.replace(/\/+$/, "")}/chat/completions`,
+    {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model: request.ai_model,
+        messages: [
+          {
+            role: "system",
+            content:
+              "You judge whether previous route semantics can be reused from privacy-safe evidence.",
+          },
+          {
+            role: "user",
+            content: buildAiReuseDecisionPrompt({
+              route,
+              previousRoute,
+              currentHash,
+            }),
+          },
+        ],
+        response_format: { type: "json_object" },
+      }),
+    },
+  );
+  if (!response.ok) {
+    return {
+      decision: "needs_review",
+      confidence: 0,
+      reason: `AI reuse decision failed: provider_${response.status}.`,
+    };
+  }
+  const body = await response.json();
+  const content = body?.choices?.[0]?.message?.content;
+  if (typeof content !== "string" || content.trim() === "") {
+    return {
+      decision: "needs_review",
+      confidence: 0,
+      reason: "AI reuse decision returned empty content.",
+    };
+  }
+  try {
+    return normalizeReuseDecision({
+      parsed: JSON.parse(content),
+      operationSourceKey: route.operation_source_key,
+    });
+  } catch {
+    return {
+      decision: "needs_review",
+      confidence: 0,
+      reason: "AI reuse decision returned malformed JSON.",
+    };
+  }
+};
+
 const requireAiProviderApiKey = (env) => {
   const apiKey = env.AI_PROVIDER_API_KEY;
   if (!apiKey) {
@@ -209,6 +371,82 @@ const callAiProvider = async ({ request, apiKey, routes }) => {
   );
 };
 
+const semanticGenerationFailureReason = (error) => {
+  if (!(error instanceof Error)) {
+    return "AI semantic generation failed for this route.";
+  }
+  const providerStatusMatch = /AI provider failed: (?<status>\d+)/.exec(
+    error.message,
+  );
+  if (providerStatusMatch?.groups?.status) {
+    return `AI semantic generation failed for this route: provider_${providerStatusMatch.groups.status}.`;
+  }
+  if (/empty semantic generation content/.test(error.message)) {
+    return "AI semantic generation returned empty content for this route.";
+  }
+  if (error instanceof SyntaxError) {
+    return "AI semantic generation returned malformed JSON for this route.";
+  }
+  return "AI semantic generation failed for this route.";
+};
+
+const unavailableAiRoute = (operationSourceKey, confidenceReason) => ({
+  operation_source_key: operationSourceKey,
+  confidence_reason: confidenceReason,
+});
+
+const assertSnapshotRouteCoverage = ({ routes, snapshotRoutes }) => {
+  const expectedKeys = routes.map((route) => route.operation_source_key);
+  const actualKeys = snapshotRoutes.map((route) => route.operation_source_key);
+  const expected = new Set(expectedKeys);
+  const actual = new Set(actualKeys);
+  const missing = expectedKeys.filter((key) => !actual.has(key));
+  const extra = actualKeys.filter((key) => !expected.has(key));
+  if (
+    expectedKeys.length !== actualKeys.length ||
+    missing.length > 0 ||
+    extra.length > 0
+  ) {
+    throw new Error(
+      `semantic snapshot route coverage mismatch: missing=${missing.join(",") || "none"} extra=${extra.join(",") || "none"}`,
+    );
+  }
+};
+
+const generateAiRoutesPerRoute = async ({ request, apiKey, routes }) => {
+  const aiRoutes = new Map();
+  for (const route of routes) {
+    try {
+      const routeAiRoutes = await callAiProvider({
+        request,
+        apiKey,
+        routes: [route],
+      });
+      const aiRoute = routeAiRoutes.get(route.operation_source_key);
+      if (aiRoute?.semantics) {
+        aiRoutes.set(route.operation_source_key, aiRoute);
+      } else {
+        aiRoutes.set(
+          route.operation_source_key,
+          unavailableAiRoute(
+            route.operation_source_key,
+            "AI semantic generation omitted this route.",
+          ),
+        );
+      }
+    } catch (error) {
+      aiRoutes.set(
+        route.operation_source_key,
+        unavailableAiRoute(
+          route.operation_source_key,
+          semanticGenerationFailureReason(error),
+        ),
+      );
+    }
+  }
+  return aiRoutes;
+};
+
 const buildAiSelectionPrompt = (selectionCandidates) =>
   JSON.stringify({
     task: "Choose adopted semantic values from deterministic and AI candidates. Return JSON only.",
@@ -1392,7 +1630,15 @@ const buildOperationAssignmentCollections = ({
   };
 };
 
-const buildSnapshot = ({ request, routes, aiRoutes, aiSelectionDecisions }) => {
+const buildSnapshot = ({
+  request,
+  routes,
+  aiRoutes,
+  aiSelectionDecisions,
+  routePlans = new Map(),
+  previousSnapshot,
+  deletedRouteCount = 0,
+}) => {
   const generatedAt = new Date().toISOString();
   const hashScope = semanticSnapshotHashScope(request);
   const semanticSnapshotVersion = buildSemanticSnapshotVersion({
@@ -1405,22 +1651,69 @@ const buildSnapshot = ({ request, routes, aiRoutes, aiSelectionDecisions }) => {
   const aiInferenceEvidence = [];
   const semanticMeaningCandidates = [];
   const catalogByGroup = new Map();
+  let failedRouteCount = 0;
+  const reusedRouteKeys = [];
+  for (const [operationSourceKey, plan] of routePlans.entries()) {
+    if (
+      plan.origin === "unchanged_route_reused" ||
+      plan.origin === "changed_route_semantic_reused"
+    ) {
+      reusedRouteKeys.push(operationSourceKey);
+    }
+  }
+  const previousContext = reusablePreviousContext({
+    previousSnapshot,
+    reusedRouteKeys,
+  });
+  sourceEvidenceRefs.push(...previousContext.sourceEvidenceRefs);
+  targetObjectProfiles.push(...previousContext.profiles);
+  targetObjectMappings.push(...previousContext.mappings);
+  aiInferenceEvidence.push(...previousContext.aiInferenceEvidence);
+  semanticMeaningCandidates.push(...previousContext.semanticMeaningCandidates);
+  for (const entry of previousContext.catalogs) {
+    const groupKey = `${entry.concept.toLowerCase()}::${entry.business_role.toLowerCase()}`;
+    catalogByGroup.set(groupKey, { ...entry });
+  }
 
   const snapshotRoutes = routes.map((route) => {
     const routeWithVersion = {
       ...route,
       semantic_snapshot_version: semanticSnapshotVersion,
     };
+    const plan = routePlans.get(route.operation_source_key) ?? {
+      origin: "new_route_ai_generated",
+      route_input_hash: routeInputHash(route),
+    };
     const routeEvidenceRefs = buildSourceEvidenceRefs(
       routeWithVersion,
       hashScope,
     );
     sourceEvidenceRefs.push(...routeEvidenceRefs);
+    if (
+      (plan.origin === "unchanged_route_reused" ||
+        plan.origin === "changed_route_semantic_reused") &&
+      plan.previous_route
+    ) {
+      return rebasePreviousRoute({
+        previousRoute: plan.previous_route,
+        currentRoute: route,
+        semanticSnapshotVersion,
+        plan,
+      });
+    }
     const aiRoute = aiRoutes.get(route.operation_source_key);
-    const baseRoute = !aiRoute?.semantics
+    if (!aiRoute?.semantics || plan.origin === "changed_route_needs_review") {
+      failedRouteCount += 1;
+    }
+    const unavailableReason =
+      plan.origin === "changed_route_needs_review"
+        ? `changed_route_needs_review: ${plan.semantic_change_reason}`
+        : (aiRoute?.confidence_reason ??
+          "AI semantics were unavailable for this route.");
+    const baseRoute = !aiRoute?.semantics || plan.origin === "changed_route_needs_review"
       ? fallbackSemantics(
           routeWithVersion,
-          "AI semantics were unavailable for this route.",
+          unavailableReason,
           hashScope,
         )
       : {
@@ -1461,10 +1754,49 @@ const buildSnapshot = ({ request, routes, aiRoutes, aiSelectionDecisions }) => {
       unresolved_operation_effects:
         assignmentCollections.unresolvedOperationEffects,
       source_evidence_refs: routeEvidenceRefs.map((entry) => entry.id),
+      route_input_hash: plan.route_input_hash,
+      previous_route_input_hash: plan.previous_route_input_hash,
+      previous_route_semantic_hash: plan.previous_route_semantic_hash,
+      semantic_origin:
+        plan.origin === "changed_route_needs_review" && aiRoute?.semantics
+          ? "changed_route_needs_review"
+          : plan.origin,
+      semantic_change_reason: plan.semantic_change_reason,
+      previous_semantic_snapshot_version:
+        plan.previous_route?.semantic_snapshot_version,
+      route_semantic_hash: routeSemanticHash({
+        ...baseRoute,
+        operation_effects: assignmentCollections.operationEffects,
+        unresolved_operation_effects:
+          assignmentCollections.unresolvedOperationEffects,
+      }),
     };
   });
+  addUniqueBy(
+    targetObjectProfiles,
+    previousContext.profiles,
+    (entry) => entry.id,
+  );
+  addUniqueBy(
+    targetObjectMappings,
+    previousContext.mappings,
+    (entry) => entry.id,
+  );
+  addUniqueBy(
+    aiInferenceEvidence,
+    previousContext.aiInferenceEvidence,
+    (entry) => entry.id,
+  );
+  addUniqueBy(
+    semanticMeaningCandidates,
+    previousContext.semanticMeaningCandidates,
+    (entry) => entry.id,
+  );
 
-  return validateSemanticSnapshotRequest({
+  const uniqueSourceEvidenceRefs = [];
+  addUniqueBy(uniqueSourceEvidenceRefs, sourceEvidenceRefs, (entry) => entry.id);
+
+  const snapshot = validateSemanticSnapshotRequest({
     project_key: request.project_key,
     idempotency_key: sha256(
       `${request.project_key}:${request.repository.repository_id}:${request.repository.merge_commit}:${request.repository.workflow_run_id ?? generatedAt}`,
@@ -1482,16 +1814,41 @@ const buildSnapshot = ({ request, routes, aiRoutes, aiSelectionDecisions }) => {
       uncertain_routes: snapshotRoutes.filter(
         (route) => route.semantics.route_confidence < 0.5,
       ).length,
-      failed_routes: 0,
+      failed_routes: failedRouteCount,
+      routes_reused: snapshotRoutes.filter((route) =>
+        [
+          "unchanged_route_reused",
+          "changed_route_semantic_reused",
+        ].includes(route.semantic_origin),
+      ).length,
+      routes_ai_generated: snapshotRoutes.filter((route) =>
+        [
+          "new_route_ai_generated",
+          "changed_route_semantic_regenerated",
+        ].includes(route.semantic_origin),
+      ).length,
+      routes_deleted: deletedRouteCount,
+      routes_needs_review: snapshotRoutes.filter(
+        (route) => route.semantic_origin === "changed_route_needs_review",
+      ).length,
+      changed_routes_semantic_reused: snapshotRoutes.filter(
+        (route) => route.semantic_origin === "changed_route_semantic_reused",
+      ).length,
+      changed_routes_semantic_regenerated: snapshotRoutes.filter(
+        (route) =>
+          route.semantic_origin === "changed_route_semantic_regenerated",
+      ).length,
     },
     routes: snapshotRoutes,
     target_object_profiles: targetObjectProfiles,
     target_object_catalog: [...catalogByGroup.values()],
     target_object_mappings: targetObjectMappings,
-    source_evidence_refs: sourceEvidenceRefs,
+    source_evidence_refs: uniqueSourceEvidenceRefs,
     ai_inference_evidence: aiInferenceEvidence,
     semantic_meaning_candidates: semanticMeaningCandidates,
   });
+  assertSnapshotRouteCoverage({ routes, snapshotRoutes: snapshot.routes });
+  return snapshot;
 };
 
 const sanitizeText = (value, route) => {
@@ -1658,6 +2015,289 @@ const sanitizeLayerEvidence = (value, route, hashScope) => {
   };
 };
 
+const previousRouteMap = (previousSnapshot) =>
+  new Map(
+    safeArray(previousSnapshot?.routes).map((route) => [
+      route.operation_source_key,
+      route,
+    ]),
+  );
+
+const buildRouteEvidencePromptEntry = ({ route, hashScope }) => ({
+  operation_source_key: route.operation_source_key,
+  method: route.method,
+  path_template: route.path_template,
+  evidence_summaries: buildSourceEvidenceRefs(route, hashScope).map(
+    (entry, index) => ({
+      evidence_index: index,
+      kind: entry.kind,
+      summary: entry.summary,
+      evidence_strength: entry.evidence_strength,
+    }),
+  ),
+});
+
+const classifyRoutesForSnapshot = async ({
+  request,
+  routes,
+  previousSnapshot,
+  aiProviderApiKey,
+  hashScope,
+}) => {
+  const previousRoutes = previousRouteMap(previousSnapshot);
+  const plans = new Map();
+  const routesRequiringGeneration = [];
+
+  for (const route of routes) {
+    const currentHash = routeInputHash(route);
+    const previousRoute = previousRoutes.get(route.operation_source_key);
+    if (!previousRoute) {
+      plans.set(route.operation_source_key, {
+        origin: "new_route_ai_generated",
+        route_input_hash: currentHash,
+      });
+      routesRequiringGeneration.push(route);
+      continue;
+    }
+
+    if (previousRoute.route_input_hash === currentHash) {
+      plans.set(route.operation_source_key, {
+        origin: "unchanged_route_reused",
+        route_input_hash: currentHash,
+        previous_route: previousRoute,
+        previous_route_input_hash: previousRoute.route_input_hash,
+        previous_route_semantic_hash:
+          previousRoute.route_semantic_hash ?? routeSemanticHash(previousRoute),
+        semantic_change_reason:
+          "Route input hash matched the previous snapshot, so previous semantics were reused exactly.",
+      });
+      continue;
+    }
+
+    const decision = await callAiReuseDecisionProvider({
+      request,
+      apiKey: aiProviderApiKey,
+      route: buildRouteEvidencePromptEntry({ route, hashScope }),
+      previousRoute,
+      currentHash,
+    });
+    if (decision.decision === "reuse_previous") {
+      plans.set(route.operation_source_key, {
+        origin: "changed_route_semantic_reused",
+        route_input_hash: currentHash,
+        previous_route: previousRoute,
+        previous_route_input_hash: previousRoute.route_input_hash,
+        previous_route_semantic_hash:
+          previousRoute.route_semantic_hash ?? routeSemanticHash(previousRoute),
+        semantic_change_reason: sanitizeText(decision.reason, route),
+      });
+      continue;
+    }
+    if (decision.decision === "regenerate") {
+      plans.set(route.operation_source_key, {
+        origin: "changed_route_semantic_regenerated",
+        route_input_hash: currentHash,
+        previous_route: previousRoute,
+        previous_route_input_hash: previousRoute.route_input_hash,
+        previous_route_semantic_hash:
+          previousRoute.route_semantic_hash ?? routeSemanticHash(previousRoute),
+        semantic_change_reason: sanitizeText(decision.reason, route),
+      });
+      routesRequiringGeneration.push(route);
+      continue;
+    }
+    plans.set(route.operation_source_key, {
+      origin: "changed_route_needs_review",
+      route_input_hash: currentHash,
+      previous_route: previousRoute,
+      previous_route_input_hash: previousRoute.route_input_hash,
+      previous_route_semantic_hash:
+        previousRoute.route_semantic_hash ?? routeSemanticHash(previousRoute),
+      semantic_change_reason: sanitizeText(decision.reason, route),
+    });
+  }
+
+  const currentRouteKeys = new Set(
+    routes.map((route) => route.operation_source_key),
+  );
+  const deletedRouteCount = safeArray(previousSnapshot?.routes).filter(
+    (route) => !currentRouteKeys.has(route.operation_source_key),
+  ).length;
+
+  return {
+    plans,
+    routesRequiringGeneration,
+    deletedRouteCount,
+  };
+};
+
+const rebasePreviousRoute = ({
+  previousRoute,
+  currentRoute,
+  semanticSnapshotVersion,
+  plan,
+}) => {
+  const route = {
+    ...previousRoute,
+    operation_source_key: currentRoute.operation_source_key,
+    semantic_snapshot_version: semanticSnapshotVersion,
+    previous_semantic_snapshot_version:
+      previousRoute.semantic_snapshot_version ??
+      plan.previous_semantic_snapshot_version,
+    method: currentRoute.method,
+    path_template: currentRoute.path_template,
+    route_input_hash: plan.route_input_hash,
+    previous_route_input_hash: plan.previous_route_input_hash,
+    previous_route_semantic_hash: plan.previous_route_semantic_hash,
+    semantic_origin: plan.origin,
+    semantic_change_reason: plan.semantic_change_reason,
+  };
+  return {
+    ...route,
+    route_semantic_hash: routeSemanticHash(route),
+  };
+};
+
+const fieldPathMatchesKeys = ({ fieldPath, routeKeys, profileIds, mappingIds, catalogKeys }) => {
+  for (const routeKey of routeKeys) {
+    if (fieldPath.startsWith(`routes[${routeKey}]`)) {
+      return true;
+    }
+  }
+  for (const profileId of profileIds) {
+    if (fieldPath.startsWith(`target_object_profiles[${profileId}]`)) {
+      return true;
+    }
+  }
+  for (const mappingId of mappingIds) {
+    if (fieldPath.startsWith(`target_object_mappings[${mappingId}]`)) {
+      return true;
+    }
+  }
+  for (const catalogKey of catalogKeys) {
+    if (fieldPath.startsWith(`target_object_catalog[${catalogKey}]`)) {
+      return true;
+    }
+  }
+  return false;
+};
+
+const reusablePreviousContext = ({ previousSnapshot, reusedRouteKeys }) => {
+  const routeKeys = new Set(reusedRouteKeys);
+  const profiles = safeArray(previousSnapshot?.target_object_profiles).filter(
+    (profile) => routeKeys.has(profile.operation_source_key),
+  );
+  const mappings = safeArray(previousSnapshot?.target_object_mappings).filter(
+    (mapping) => routeKeys.has(mapping.operation_source_key),
+  );
+  const profileIds = new Set(profiles.map((profile) => profile.id));
+  const mappingIds = new Set(mappings.map((mapping) => mapping.id));
+  const catalogKeys = new Set([
+    ...mappings.map((mapping) => mapping.target_object_key),
+    ...safeArray(previousSnapshot?.routes)
+      .filter((route) => routeKeys.has(route.operation_source_key))
+      .flatMap((route) =>
+        safeArray(route.operation_effects).map((effect) => effect.target_object_key),
+      ),
+  ]);
+  const catalogs = safeArray(previousSnapshot?.target_object_catalog).filter(
+    (entry) => catalogKeys.has(entry.target_object_key),
+  );
+  const matches = (fieldPath) =>
+    fieldPathMatchesKeys({
+      fieldPath,
+      routeKeys,
+      profileIds,
+      mappingIds,
+      catalogKeys,
+    });
+  const aiInferenceEvidence = safeArray(
+    previousSnapshot?.ai_inference_evidence,
+  ).filter(
+    (entry) =>
+      safeArray(entry.output_field_paths).length > 0 &&
+      safeArray(entry.output_field_paths).every(matches),
+  );
+  const aiInferenceIds = new Set(aiInferenceEvidence.map((entry) => entry.id));
+  const semanticMeaningCandidates = safeArray(
+    previousSnapshot?.semantic_meaning_candidates,
+  ).filter(
+    (entry) =>
+      matches(entry.field_path) &&
+      aiInferenceIds.has(entry.ai_candidate.ai_inference_evidence_ref) &&
+      aiInferenceIds.has(entry.selection_ai_inference_evidence_ref),
+  );
+  const sourceEvidenceIds = new Set();
+  for (const route of safeArray(previousSnapshot?.routes).filter((entry) =>
+    routeKeys.has(entry.operation_source_key),
+  )) {
+    for (const ref of safeArray(route.source_evidence_refs)) {
+      sourceEvidenceIds.add(ref);
+    }
+    for (const effect of safeArray(route.operation_effects)) {
+      for (const ref of safeArray(effect.source_evidence_refs)) {
+        sourceEvidenceIds.add(ref);
+      }
+    }
+    for (const effect of safeArray(route.unresolved_operation_effects)) {
+      for (const ref of safeArray(effect.source_evidence_refs)) {
+        sourceEvidenceIds.add(ref);
+      }
+    }
+  }
+  for (const profile of profiles) {
+    for (const ref of [
+      ...safeArray(profile.evidence_refs),
+      ...safeArray(profile.source_evidence_refs),
+    ]) {
+      sourceEvidenceIds.add(ref);
+    }
+  }
+  for (const mapping of mappings) {
+    sourceEvidenceIds.add(mapping.grouping_evidence_ref);
+  }
+  for (const catalog of catalogs) {
+    for (const ref of safeArray(catalog.grouping_evidence_refs)) {
+      sourceEvidenceIds.add(ref);
+    }
+  }
+  for (const evidence of aiInferenceEvidence) {
+    for (const ref of safeArray(evidence.input_source_evidence_refs)) {
+      sourceEvidenceIds.add(ref);
+    }
+  }
+  for (const candidate of semanticMeaningCandidates) {
+    for (const ref of [
+      ...safeArray(candidate.deterministic_candidate?.source_evidence_refs),
+      ...safeArray(candidate.ai_candidate?.source_evidence_refs),
+    ]) {
+      sourceEvidenceIds.add(ref);
+    }
+  }
+  const sourceEvidenceRefs = safeArray(previousSnapshot?.source_evidence_refs).filter(
+    (entry) => sourceEvidenceIds.has(entry.id),
+  );
+  return {
+    profiles,
+    mappings,
+    catalogs,
+    sourceEvidenceRefs,
+    aiInferenceEvidence,
+    semanticMeaningCandidates,
+  };
+};
+
+const addUniqueBy = (target, entries, keyFn) => {
+  const seen = new Set(target.map(keyFn));
+  for (const entry of entries) {
+    const key = keyFn(entry);
+    if (!seen.has(key)) {
+      target.push(entry);
+      seen.add(key);
+    }
+  }
+};
+
 const FORBIDDEN_CODE_STRUCTURE_PATTERNS = [
   { label: "file path", pattern: /\b[A-Za-z0-9_./-]+\.py\b/i },
   {
@@ -1755,6 +2395,56 @@ const semanticSnapshotUrl = (baseUrl) => {
   return `${normalized}/api/v1/semantic-snapshots`;
 };
 
+const semanticSnapshotLatestUrl = (request) => {
+  const url = new URL(`${semanticSnapshotUrl(request.clue_api_base_url)}/latest`);
+  url.searchParams.set("project_key", request.project_key);
+  url.searchParams.set("environment", request.environment);
+  url.searchParams.set("repository_id", request.repository.repository_id);
+  url.searchParams.set("service_key", request.service.service_key);
+  return url.toString();
+};
+
+const fetchLatestSnapshot = async ({ request, env }) => {
+  const apiKey = env.CLUE_API_KEY;
+  if (!apiKey) {
+    return null;
+  }
+  let response;
+  try {
+    response = await fetch(semanticSnapshotLatestUrl(request), {
+      method: "GET",
+      headers: {
+        authorization: `Bearer ${apiKey}`,
+      },
+    });
+  } catch {
+    return null;
+  }
+  if (response.status === 404 || response.status === 204) {
+    return null;
+  }
+  if (!response.ok) {
+    return null;
+  }
+  const body = await response.json();
+  const candidate = body?.snapshot ?? body;
+  if (!candidate || !Array.isArray(candidate.routes)) {
+    return null;
+  }
+  const parsed = validateSemanticSnapshotRequest(candidate);
+  assertNoRawCodeStructure(parsed);
+  return parsed;
+};
+
+const validatePreviousSnapshotForReuse = (snapshot) => {
+  if (!snapshot) {
+    return null;
+  }
+  const parsed = validateSemanticSnapshotRequest(snapshot);
+  assertNoRawCodeStructure(parsed);
+  return parsed;
+};
+
 const sendSnapshot = async ({ request, env, snapshot }) => {
   const apiKey = env.CLUE_API_KEY;
   if (!apiKey) {
@@ -1773,10 +2463,24 @@ const sendSnapshot = async ({ request, env, snapshot }) => {
   if (!response.ok) {
     throw new Error(`Clue semantic snapshot upload failed: ${response.status}`);
   }
-  return response.json();
+  const body = await response.json();
+  try {
+    return validateSemanticSnapshotResponse(body);
+  } catch (error) {
+    throw new Error(
+      `Clue semantic snapshot upload response is invalid: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
 };
 
-export const runSemanticCi = async ({ repoRoot, request: rawRequest, env }) => {
+export const runSemanticCi = async ({
+  repoRoot,
+  request: rawRequest,
+  env,
+  previousSnapshot: providedPreviousSnapshot,
+}) => {
   const request = validateSemanticCiRequest(rawRequest);
   const hashScope = semanticSnapshotHashScope(request);
   const files = await listAllowedPythonFiles({
@@ -1785,41 +2489,59 @@ export const runSemanticCi = async ({ repoRoot, request: rawRequest, env }) => {
     excludedSourcePaths: request.excluded_source_paths,
   });
   const routes = await analyzeFastApiRoutes({ repoRoot, files });
-  const promptRoutes = routes.map((route) => ({
-    operation_source_key: route.operation_source_key,
-    method: route.method,
-    path_template: route.path_template,
-    evidence_summaries: buildSourceEvidenceRefs(route, hashScope).map(
-      (entry, index) => ({
-        evidence_index: index,
-        kind: entry.kind,
-        summary: entry.summary,
-        evidence_strength: entry.evidence_strength,
-      }),
-    ),
-  }));
-  const aiProviderApiKey = requireAiProviderApiKey(env);
-  const aiRoutes = await callAiProvider({
+  if (routes.length === 0) {
+    throw new Error(
+      "semantic CI discovered zero routes; check framework, backend root path, and allowed_source_paths",
+    );
+  }
+  const previousSnapshot =
+    providedPreviousSnapshot === undefined
+      ? await fetchLatestSnapshot({ request, env })
+      : validatePreviousSnapshotForReuse(providedPreviousSnapshot);
+  const previousRoutes = previousRouteMap(previousSnapshot);
+  const routeNeedsAi = routes.some((route) => {
+    const previousRoute = previousRoutes.get(route.operation_source_key);
+    return !previousRoute || previousRoute.route_input_hash !== routeInputHash(route);
+  });
+  const aiProviderApiKey = routeNeedsAi ? requireAiProviderApiKey(env) : null;
+  const { plans: routePlans, routesRequiringGeneration, deletedRouteCount } =
+    await classifyRoutesForSnapshot({
+      request,
+      routes,
+      previousSnapshot,
+      aiProviderApiKey,
+      hashScope,
+    });
+  const promptRoutes = routesRequiringGeneration.map((route) =>
+    buildRouteEvidencePromptEntry({ route, hashScope }),
+  );
+  const aiRoutes = await generateAiRoutesPerRoute({
     request,
     apiKey: aiProviderApiKey,
     routes: promptRoutes,
   });
   const { selectionCandidates, routeBySelectionKey } = buildSelectorCandidates({
-    routes,
+    routes: routesRequiringGeneration,
     aiRoutes,
     hashScope,
   });
-  const aiSelectionDecisions = await callAiSelectionProvider({
-    request,
-    apiKey: aiProviderApiKey,
-    selectionCandidates,
-    routeBySelectionKey,
-  });
+  const aiSelectionDecisions =
+    selectionCandidates.length === 0
+      ? new Map()
+      : await callAiSelectionProvider({
+          request,
+          apiKey: aiProviderApiKey,
+          selectionCandidates,
+          routeBySelectionKey,
+        });
   const snapshot = buildSnapshot({
     request,
     routes,
     aiRoutes,
     aiSelectionDecisions,
+    routePlans,
+    previousSnapshot,
+    deletedRouteCount,
   });
   const upload = await sendSnapshot({ request, env, snapshot });
   return {
@@ -1828,3 +2550,30 @@ export const runSemanticCi = async ({ repoRoot, request: rawRequest, env }) => {
     upload,
   };
 };
+
+export const runSemanticInventory = async ({ repoRoot, request }) => {
+  const files = await listAllowedPythonFiles({
+    repoRoot,
+    allowedSourcePaths: request.allowed_source_paths,
+    excludedSourcePaths: request.excluded_source_paths,
+  });
+  const routes = await analyzeFastApiRoutes({ repoRoot, files });
+  if (routes.length === 0) {
+    throw new Error(
+      "semantic inventory discovered zero routes; check framework, backend root path, and allowed_source_paths",
+    );
+  }
+  return {
+    framework: request.framework,
+    service_key: request.service_key,
+    allowed_source_paths: request.allowed_source_paths,
+    excluded_source_paths: request.excluded_source_paths,
+    route_count: routes.length,
+    operation_source_keys: routes.map((route) => route.operation_source_key),
+    routes: routes.map((route) => ({
+      operation_source_key: route.operation_source_key,
+      method: route.method,
+      path_template: route.path_template,
+    })),
+  };
+};