@opengeni/core 0.28.1 → 1.0.0

package/src/domain/insights.ts

@@ -5,38 +5,82 @@ import {
   type WorkspaceInsightsResponse,
 } from "@opengeni/contracts";
 import {
-  aggregateModelCallFacts,
-  aggregateModelCallFactsByDay,
-  aggregateModelCallFactsByHour,
-  aggregateRootSessionDrivers,
-  aggregateScheduleFacts,
   aggregateSessionDepth,
-  aggregateWarmSecondsByGroup,
   countOnlineMachines,
   countScheduledTaskFires,
   countSessionsAttachedToGroups,
   enumerateUtcDays,
   enumerateUtcHours,
+  getSessionAccessProjection,
   listFloorSessions,
   listLiveWarmLeases,
-  listModelCallFacets,
-  listRecentModelCalls,
   listScheduledTasks,
+  readWorkspaceInsightsModelBundle,
+  readWorkspaceInsightsUsageBundle,
   requireWorkspace,
-  sumUsageQuantity,
-  sumUsageQuantityByDay,
-  sumUsageQuantityByHour,
-  sumUsageQuantityInRange,
   type Database,
 } from "@opengeni/db";
 
+import { measureInsightsPhase, type InsightsPhaseObserver } from "./insights-timing";
+export { measureInsightsPhase, workspaceInsightsPhaseMetricObserver } from "./insights-timing";
+export type { InsightsPhaseObservation, InsightsPhaseObserver } from "./insights-timing";
 const MACHINE_HEARTBEAT_FRESH_MS = 120_000;
+export const WORKSPACE_INSIGHTS_PROVIDER_FILTER_MAX_UTF8_BYTES = 256;
+export const WORKSPACE_INSIGHTS_MODEL_FILTER_MAX_UTF8_BYTES = 512;
+
+export type WorkspaceInsightsFilterField = "provider" | "model" | "rootSessionId" | "sessionId";
+
+export class WorkspaceInsightsFilterValidationError extends Error {
+  readonly field: WorkspaceInsightsFilterField;
+  readonly maxUtf8Bytes: number;
+
+  constructor(field: WorkspaceInsightsFilterField, maxUtf8Bytes: number, message?: string) {
+    super(message ?? `${field} must be at most ${maxUtf8Bytes} UTF-8 bytes`);
+    this.name = "WorkspaceInsightsFilterValidationError";
+    this.field = field;
+    this.maxUtf8Bytes = maxUtf8Bytes;
+  }
+}
+
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/** Normalize a session drilldown scope; only exact UUIDs are accepted. */
+export function normalizeWorkspaceInsightsSessionScope(
+  value: string | null | undefined,
+  field: "rootSessionId" | "sessionId",
+): string | null {
+  const normalized = value?.trim() || null;
+  if (normalized === null || normalized === "all") return null;
+  if (!UUID_PATTERN.test(normalized)) {
+    throw new WorkspaceInsightsFilterValidationError(field, 36, `${field} must be a UUID`);
+  }
+  return normalized.toLowerCase();
+}
+
+/** Normalize the public filter sentinel and enforce the database authority envelope. */
+export function normalizeWorkspaceInsightsFilter(
+  value: string | null | undefined,
+  field: WorkspaceInsightsFilterField,
+): string | null {
+  const normalized = value?.trim() || null;
+  if (normalized === null || normalized === "all") return null;
+  const maxUtf8Bytes =
+    field === "provider"
+      ? WORKSPACE_INSIGHTS_PROVIDER_FILTER_MAX_UTF8_BYTES
+      : WORKSPACE_INSIGHTS_MODEL_FILTER_MAX_UTF8_BYTES;
+  if (new TextEncoder().encode(normalized).byteLength > maxUtf8Bytes) {
+    throw new WorkspaceInsightsFilterValidationError(field, maxUtf8Bytes);
+  }
+  return normalized;
+}
 
 export type GetWorkspaceInsightsInput = {
   workspaceId: string;
   range: InsightsRange;
   provider?: string | null;
   model?: string | null;
+  rootSessionId?: string | null;
+  sessionId?: string | null;
   now?: Date;
 };
 
@@ -173,171 +217,150 @@ function pricingSourceOf(
   return value === "configured_list_price" || value === "gateway_reported" ? value : null;
 }
 
+export function insightsSessionLabel(input: {
+  id: string;
+  title: string | null;
+  depth?: number | null;
+}): string {
+  const title = input.title?.trim();
+  if (title) return title;
+  return `${(input.depth ?? 0) > 0 ? "Agent" : "Session"} ${input.id.slice(0, 8)}`;
+}
+
+export function insightsProjectLabel(input: {
+  kind: "project" | "other" | "unfiled" | "unavailable" | "deleted";
+  name: string | null;
+  projects: number;
+}): string {
+  switch (input.kind) {
+    case "project":
+      return input.name?.trim() || "Untitled project";
+    case "other":
+      return `${input.projects.toLocaleString("en-US")} other ${input.projects === 1 ? "project" : "projects"}`;
+    case "unfiled":
+      return "No project";
+    case "unavailable":
+      return "Root session not visible";
+    case "deleted":
+      return "Deleted chats";
+  }
+}
+
 export async function getWorkspaceInsights(
   db: Database,
   settings: Settings,
   input: GetWorkspaceInsightsInput,
+  observePhase?: InsightsPhaseObserver,
 ): Promise<WorkspaceInsightsResponse> {
-  await requireWorkspace(db, input.workspaceId);
+  const provider = normalizeWorkspaceInsightsFilter(input.provider, "provider");
+  const model = normalizeWorkspaceInsightsFilter(input.model, "model");
+  const rootSessionId = normalizeWorkspaceInsightsSessionScope(
+    input.rootSessionId,
+    "rootSessionId",
+  );
+  const sessionId = normalizeWorkspaceInsightsSessionScope(input.sessionId, "sessionId");
+  await measureInsightsPhase(observePhase, "require_workspace", () =>
+    requireWorkspace(db, input.workspaceId),
+  );
+  const [visibleRootScope, visibleSessionScope] = await Promise.all([
+    rootSessionId ? getSessionAccessProjection(db, input.workspaceId, rootSessionId) : null,
+    sessionId ? getSessionAccessProjection(db, input.workspaceId, sessionId) : null,
+  ]);
   const now = input.now ?? new Date();
   const window = resolveRangeWindow(input.range, now);
-  const provider = input.provider?.trim() || null;
-  const model = input.model?.trim() || null;
   const modelFilterActive = Boolean(provider || model);
-  const filter = { provider, model };
-  const aggregateFactsForSeries =
-    input.range === "today" ? aggregateModelCallFactsByHour : aggregateModelCallFactsByDay;
-  const sumUsageForSeries =
-    input.range === "today" ? sumUsageQuantityByHour : sumUsageQuantityByDay;
+  const scopeActive = Boolean(rootSessionId || sessionId);
+  const filter = { provider, model, rootSessionId, sessionId };
+
+  const [modelBundle, usageBundle, liveWarm, tasks, depth, floorRows, machinesOnline] =
+    await Promise.all([
+      measureInsightsPhase(observePhase, "model_bundle", () =>
+        readWorkspaceInsightsModelBundle(db, {
+          workspaceId: input.workspaceId,
+          since: window.since,
+          until: window.until,
+          priorSince: window.priorSince,
+          priorUntil: window.priorUntil,
+          granularity: input.range === "today" ? "hour" : "day",
+          ...filter,
+        }),
+      ),
+      measureInsightsPhase(observePhase, "usage_bundle", () =>
+        readWorkspaceInsightsUsageBundle(db, {
+          workspaceId: input.workspaceId,
+          since: window.since,
+          until: window.until,
+          priorSince: window.priorSince,
+          priorUntil: window.priorUntil,
+          monthSince: startOfUtcMonth(now),
+          granularity: input.range === "today" ? "hour" : "day",
+          warmGroupLimit: 24,
+        }),
+      ),
+      measureInsightsPhase(observePhase, "live_warm", () =>
+        listLiveWarmLeases(db, input.workspaceId),
+      ),
+      measureInsightsPhase(observePhase, "scheduled_tasks", () =>
+        listScheduledTasks(db, input.workspaceId, 100),
+      ),
+      measureInsightsPhase(observePhase, "session_depth", () =>
+        aggregateSessionDepth(db, input.workspaceId),
+      ),
+      measureInsightsPhase(observePhase, "floor_sessions", () =>
+        listFloorSessions(db, input.workspaceId, 24),
+      ),
+      settings.sandboxSelfhostedEnabled
+        ? measureInsightsPhase(observePhase, "online_machines", () =>
+            countOnlineMachines(db, input.workspaceId, MACHINE_HEARTBEAT_FRESH_MS, now),
+          )
+        : Promise.resolve(0),
+    ]);
 
-  const [
+  const {
+    modelRows,
+    priorModelRows,
+    factBuckets: factDays,
+    rootDrivers,
+    priorRootDrivers,
+    projects,
+    privateChats,
+    privateChatsTruncated,
+    scheduleFacts,
+    facets,
+    recentCalls,
+    promptContributions,
+    dataThrough,
+    driverGroups,
+    driversTruncated,
+    facetsTruncated,
+    recentCallsTruncated,
+  } = modelBundle;
+  const {
     workspaceCreditMicros,
     priorWorkspaceCreditMicros,
     warmSeconds,
     priorWarmSeconds,
-    modelRows,
-    priorModelRows,
-    factDays,
-    warmDays,
-    costDays,
+    buckets: usageBuckets,
     warmGroups,
-    liveWarm,
-    rootDrivers,
-    scheduleFacts,
-    tasks,
-    depth,
-    floorRows,
-    machinesOnline,
     billableTokensUsed,
     agentRunsUsed,
-    facets,
-    recentCalls,
-  ] = await Promise.all([
-    sumUsageQuantityInRange(db, {
-      workspaceId: input.workspaceId,
-      eventType: "model.cost",
-      since: window.since,
-      until: window.until,
-    }),
-    sumUsageQuantityInRange(db, {
-      workspaceId: input.workspaceId,
-      eventType: "model.cost",
-      since: window.priorSince,
-      until: window.priorUntil,
-    }),
-    sumUsageQuantityInRange(db, {
-      workspaceId: input.workspaceId,
-      eventType: "sandbox.warm_seconds",
-      since: window.since,
-      until: window.until,
-    }),
-    sumUsageQuantityInRange(db, {
-      workspaceId: input.workspaceId,
-      eventType: "sandbox.warm_seconds",
-      since: window.priorSince,
-      until: window.priorUntil,
-    }),
-    aggregateModelCallFacts(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      ...filter,
-    }),
-    aggregateModelCallFacts(db, {
-      workspaceId: input.workspaceId,
-      since: window.priorSince,
-      until: window.priorUntil,
-      ...filter,
-    }),
-    aggregateFactsForSeries(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      ...filter,
-    }),
-    sumUsageForSeries(db, {
-      workspaceId: input.workspaceId,
-      eventType: "sandbox.warm_seconds",
-      since: window.since,
-      until: window.until,
-    }),
-    sumUsageForSeries(db, {
-      workspaceId: input.workspaceId,
-      eventType: "model.cost",
-      since: window.since,
-      until: window.until,
-    }),
-    aggregateWarmSecondsByGroup(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      limit: 24,
-    }),
-    listLiveWarmLeases(db, input.workspaceId),
-    aggregateRootSessionDrivers(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      ...filter,
-      limit: 8,
-    }),
-    aggregateScheduleFacts(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      ...filter,
-    }),
-    listScheduledTasks(db, input.workspaceId, 100),
-    aggregateSessionDepth(db, input.workspaceId),
-    listFloorSessions(db, input.workspaceId, 24),
-    settings.sandboxSelfhostedEnabled
-      ? countOnlineMachines(db, input.workspaceId, MACHINE_HEARTBEAT_FRESH_MS, now)
-      : Promise.resolve(0),
-    sumUsageQuantity(db, {
-      workspaceId: input.workspaceId,
-      eventType: "model.tokens",
-      since: startOfUtcMonth(now),
-    }),
-    sumUsageQuantity(db, {
-      workspaceId: input.workspaceId,
-      eventType: "agent_run.created",
-      since: startOfUtcMonth(now),
-    }),
-    listModelCallFacets(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-    }),
-    listRecentModelCalls(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      ...filter,
-      limit: 50,
-    }),
-  ]);
-
-  const [priorRootDrivers, attached, fireCounts] = await Promise.all([
-    // Exact prior costs for the current top drivers — never a separate top-N page that
-    // drops roots and invents +$full as "new" spend.
-    aggregateRootSessionDrivers(db, {
-      workspaceId: input.workspaceId,
-      since: window.priorSince,
-      until: window.priorUntil,
-      ...filter,
-      rootSessionIds: rootDrivers.map((row) => row.rootSessionId),
-    }),
-    countSessionsAttachedToGroups(
-      db,
-      input.workspaceId,
-      warmGroups.map((group) => group.groupId),
+  } = usageBundle;
+  const [attached, fireCounts] = await Promise.all([
+    measureInsightsPhase(observePhase, "attached_sessions", () =>
+      countSessionsAttachedToGroups(
+        db,
+        input.workspaceId,
+        warmGroups.map((group) => group.groupId),
+      ),
+    ),
+    measureInsightsPhase(observePhase, "scheduled_fires", () =>
+      countScheduledTaskFires(db, {
+        workspaceId: input.workspaceId,
+        since: window.since,
+        until: window.until,
+        taskIds: tasks.map((task) => task.id),
+      }),
     ),
-    countScheduledTaskFires(db, {
-      workspaceId: input.workspaceId,
-      since: window.since,
-      until: window.until,
-      taskIds: tasks.map((task) => task.id),
-    }),
   ]);
   const backendByGroup = new Map(liveWarm.map((lease) => [lease.groupId, lease.backend]));
   const warmSecondsByGroup = new Map(warmGroups.map((group) => [group.groupId, group.warmSeconds]));
@@ -361,6 +384,8 @@ export async function getWorkspaceInsights(
       creditUsd: microsToUsd(row.pricedCostMicros),
       estimatedProviderUsd: microsToUsd(row.estimatedProviderCostMicros),
       estimatedProviderCostKnownCalls: row.estimatedProviderCostKnownCalls,
+      equivalentCreditUsd: microsToUsd(row.equivalentCreditCostMicros),
+      equivalentCreditCostKnownCalls: row.equivalentCreditCostKnownCalls,
     }))
     .sort((a, b) => b.totalTokens - a.totalTokens);
 
@@ -382,7 +407,25 @@ export async function getWorkspaceInsights(
     (sum, row) => sum + row.estimatedProviderCostKnownCalls,
     0,
   );
+  const equivalentCreditCostMicros = modelRows.reduce(
+    (sum, row) => sum + row.equivalentCreditCostMicros,
+    0,
+  );
+  const priorEquivalentCreditCostMicros = priorModelRows.reduce(
+    (sum, row) => sum + row.equivalentCreditCostMicros,
+    0,
+  );
+  const equivalentCreditCostKnownCalls = modelRows.reduce(
+    (sum, row) => sum + row.equivalentCreditCostKnownCalls,
+    0,
+  );
+  const priorEquivalentCreditCostKnownCalls = priorModelRows.reduce(
+    (sum, row) => sum + row.equivalentCreditCostKnownCalls,
+    0,
+  );
   const modelCalls = modelRows.reduce((sum, row) => sum + row.calls, 0);
+  const cachedTokens = modelRows.reduce((sum, row) => sum + row.cachedTokens, 0);
+  const cacheInputTokens = modelRows.reduce((sum, row) => sum + row.cacheInputTokens, 0);
   const priorInputTokens = priorModelRows.reduce((sum, row) => sum + row.inputTokens, 0);
   const priorTotalTokens = priorModelRows.reduce((sum, row) => sum + row.totalTokens, 0);
   const priorCachedTokens = priorModelRows.reduce((sum, row) => sum + row.cachedTokens, 0);
@@ -398,6 +441,8 @@ export async function getWorkspaceInsights(
       costMicros: 0,
       estimatedProviderCostMicros: 0,
       estimatedProviderCostKnownCalls: 0,
+      equivalentCreditCostMicros: 0,
+      equivalentCreditCostKnownCalls: 0,
       inputTokens: 0,
       outputTokens: 0,
       cachedTokens: 0,
@@ -409,15 +454,18 @@ export async function getWorkspaceInsights(
       cacheKnownCalls: 0,
       calls: 0,
     };
-    const modelCostMicros = modelFilterActive
-      ? facts.costMicros
-      : (costDays.get(bucket) ?? facts.costMicros);
+    const modelCostMicros =
+      modelFilterActive || scopeActive
+        ? facts.costMicros
+        : (usageBuckets.get(bucket)?.costMicros ?? facts.costMicros);
     return {
       label: input.range === "today" ? bucket.slice(11) : bucket.slice(5),
       modelCostUsd: microsToUsd(modelCostMicros),
       estimatedProviderUsd: microsToUsd(facts.estimatedProviderCostMicros),
       estimatedProviderCostKnownCalls: facts.estimatedProviderCostKnownCalls,
-      warmSeconds: warmDays.get(bucket) ?? 0,
+      equivalentCreditUsd: microsToUsd(facts.equivalentCreditCostMicros),
+      equivalentCreditCostKnownCalls: facts.equivalentCreditCostKnownCalls,
+      warmSeconds: usageBuckets.get(bucket)?.warmSeconds ?? 0,
       inputTokens: facts.inputTokens,
       outputTokens: facts.outputTokens,
       cachedTokens: facts.cachedTokens,
@@ -445,10 +493,12 @@ export async function getWorkspaceInsights(
     return {
       id: `root:${row.rootSessionId}`,
       groupBy: "root_session" as const,
-      label: row.title?.trim() || row.rootSessionId.slice(0, 8),
+      label: insightsSessionLabel({ id: row.rootSessionId, title: row.title, depth: 0 }),
       creditUsd,
       estimatedProviderUsd: microsToUsd(row.estimatedProviderCostMicros),
       estimatedProviderCostKnownCalls: row.estimatedProviderCostKnownCalls,
+      equivalentCreditUsd: microsToUsd(row.equivalentCreditCostMicros),
+      equivalentCreditCostKnownCalls: row.equivalentCreditCostKnownCalls,
       tokens: row.totalTokens,
       cacheHitPct: cacheHitPct(row.cachedTokens, row.cacheInputTokens),
       pctOfCreditUsd:
@@ -460,6 +510,20 @@ export async function getWorkspaceInsights(
     };
   });
 
+  const projectRows = projects.map((row) => ({
+    id: row.kind === "project" && row.channelId ? `project:${row.channelId}` : row.kind,
+    kind: row.kind,
+    label: insightsProjectLabel(row),
+    projects: row.projects,
+    rootSessions: row.rootSessions,
+    calls: row.calls,
+    creditUsd: microsToUsd(row.pricedCostMicros),
+    estimatedProviderUsd: microsToUsd(row.estimatedProviderCostMicros),
+    estimatedProviderCostKnownCalls: row.estimatedProviderCostKnownCalls,
+    tokens: row.totalTokens,
+    cacheHitPct: cacheHitPct(row.cachedTokens, row.cacheInputTokens),
+  }));
+
   const scheduleFactById = new Map(scheduleFacts.map((row) => [row.scheduledTaskId, row] as const));
   const schedules = tasks.map((task) => {
     const fact = scheduleFactById.get(task.id);
@@ -470,6 +534,8 @@ export async function getWorkspaceInsights(
       creditUsd: fact ? microsToUsd(fact.pricedCostMicros) : null,
       estimatedProviderUsd: fact ? microsToUsd(fact.estimatedProviderCostMicros) : null,
       estimatedProviderCostKnownCalls: fact ? fact.estimatedProviderCostKnownCalls : null,
+      equivalentCreditUsd: fact ? microsToUsd(fact.equivalentCreditCostMicros) : null,
+      equivalentCreditCostKnownCalls: fact ? fact.equivalentCreditCostKnownCalls : null,
       tokens: fact ? fact.totalTokens : null,
       cacheHitPct: fact ? cacheHitPct(fact.cachedTokens, fact.cacheInputTokens) : null,
       billing: fact ? billingPathOf(fact.billingPath) : null,
@@ -486,7 +552,11 @@ export async function getWorkspaceInsights(
     })
     .map((row) => ({
       id: row.id,
-      title: row.title?.trim() || "Untitled session",
+      title: insightsSessionLabel({
+        id: row.id,
+        title: row.title,
+        depth: row.nestedAgentDepth,
+      }),
       state: floorState(row),
       depth: row.nestedAgentDepth,
       model: row.model,
@@ -514,13 +584,20 @@ export async function getWorkspaceInsights(
       sessions: bucket.sessions,
     })),
     drivers,
+    projects: projectRows,
+    privateChats,
+    privateChatsTruncated,
     schedules,
     recentCalls: recentCalls.map((row) => ({
       id: row.id,
       occurredAt: row.occurredAt.toISOString(),
       recordedAt: row.recordedAt.toISOString(),
       sessionId: row.sessionId,
-      sessionTitle: row.sessionTitle?.trim() || "Untitled session",
+      sessionTitle: insightsSessionLabel({
+        id: row.sessionId,
+        title: row.sessionTitle,
+        depth: row.sessionDepth,
+      }),
       turnId: row.turnId,
       provider: row.provider,
       providerApi: row.providerApi,
@@ -537,8 +614,11 @@ export async function getWorkspaceInsights(
         row.estimatedProviderCostMicros == null
           ? null
           : microsToUsd(row.estimatedProviderCostMicros),
+      equivalentCreditUsd:
+        row.equivalentCreditCostMicros == null ? null : microsToUsd(row.equivalentCreditCostMicros),
       pricingSource: pricingSourceOf(row.pricingSource),
     })),
+    promptContributions,
     warmSeconds,
     priorWarmSeconds,
     warmGroups: warmGroups.map((group) => ({
@@ -569,24 +649,45 @@ export async function getWorkspaceInsights(
     priorEstimatedProviderUsd: microsToUsd(priorEstimatedProviderCostMicros),
     estimatedProviderCostKnownCalls,
     priorEstimatedProviderCostKnownCalls,
+    equivalentCreditUsd: microsToUsd(equivalentCreditCostMicros),
+    priorEquivalentCreditUsd: microsToUsd(priorEquivalentCreditCostMicros),
+    equivalentCreditCostKnownCalls,
+    priorEquivalentCreditCostKnownCalls,
     modelCalls,
     priorInputTokens,
     priorTotalTokens,
     priorCacheHitPct: cacheHitPct(priorCachedTokens, priorCacheInputTokens),
+    priorCacheInputTokens,
     priorCalls,
     goalsActive: depth.goalsActive,
     goalsCompleted: depth.goalsCompleted,
     sessionsTouched: depth.sessionsTouched,
     rootSessions: depth.rootSessions,
     deepestDepth: depth.deepestDepth,
-    deepestSessionTitle: depth.deepestSessionTitle || "",
+    deepestSessionTitle: depth.deepestSessionId
+      ? insightsSessionLabel({
+          id: depth.deepestSessionId,
+          title: depth.deepestSessionTitle,
+          depth: depth.deepestDepth,
+        })
+      : "",
     avgDepth: Math.round(depth.avgDepth * 10) / 10,
     warmIdleNow: liveWarm.filter((lease) => lease.turnHolders === 0).length,
     billableTokensUsed,
     billableTokenCap: limits.maxMonthlyTokensPerWorkspace ?? null,
     agentRunsUsed,
     agentRunCap: limits.maxMonthlyAgentRunsPerWorkspace ?? null,
-    modelFilterActive,
+    modelFilterActive: modelFilterActive || scopeActive,
+    dataThrough: dataThrough?.toISOString() ?? null,
+    cacheHitPct: cacheHitPct(cachedTokens, cacheInputTokens),
+    scope: {
+      rootSessionId: visibleRootScope ? rootSessionId : null,
+      sessionId: visibleSessionScope ? sessionId : null,
+    },
+    driverGroups,
+    driversTruncated,
+    facetsTruncated,
+    recentCallsTruncated,
   });
 
   return { snapshot };

package/src/domain/knowledge-files.ts

@@ -0,0 +1,46 @@
+import { createHash } from "node:crypto";
+import {
+  completeKnowledgeFilePreparation,
+  inspectKnowledgeFilePreparation,
+  type KnowledgeContext,
+} from "@opengeni/db";
+import { retryWhileMissing } from "@opengeni/storage";
+import type { ApiRouteDeps } from "../dependencies";
+
+/** Existing original -> one source revision, through the ordinary task policy. */
+export async function prepareKnowledgeFile(
+  deps: Pick<ApiRouteDeps, "db" | "objectStorage" | "getDocumentServices">,
+  context: KnowledgeContext,
+  fileId: string,
+  purpose: "evidence" | "reference" = "evidence",
+) {
+  const inspected = await inspectKnowledgeFilePreparation(deps.db, context, fileId);
+  if (inspected.status !== "prepare") return inspected;
+  if (!deps.objectStorage) throw new Error("File storage is unavailable");
+  const object = await retryWhileMissing(() =>
+    deps.objectStorage!.getObjectBytes(inspected.file.objectKey),
+  );
+  if (!object) throw new Error("The original file is unavailable");
+  const sourceVersion = createHash("sha256").update(object.bytes).digest("hex");
+  if (
+    object.bytes.byteLength !== inspected.file.sizeBytes ||
+    (inspected.file.sha256 && inspected.file.sha256.toLowerCase() !== sourceVersion)
+  ) {
+    throw new Error("The original file no longer matches its retained metadata");
+  }
+  // A screenshot/diagram can support a visual finding without containing text.
+  // Keep its exact original; OCR is neither required evidence nor the finding.
+  const originalOnly = purpose === "evidence" && inspected.file.contentType.startsWith("image/");
+  const parsed = originalOnly
+    ? { text: "" }
+    : await deps.getDocumentServices().parser.parse(object.bytes, inspected.file);
+  if (!originalOnly && !parsed.text.trim())
+    throw new Error("No searchable text could be extracted from this file");
+  return completeKnowledgeFilePreparation(deps.db, context, {
+    fileId,
+    title: inspected.file.filename,
+    sourceVersion,
+    content: parsed.text,
+    purpose,
+  });
+}