@opengeni/core 0.28.0 → 1.0.0

package/src/domain/knowledge-messages.test.ts

@@ -0,0 +1,91 @@
+import { afterAll, beforeEach, expect, mock, test } from "bun:test";
+const actor = {
+  kind: "agent",
+  sessionId: "session-a",
+  turnId: "turn-a",
+  attemptId: "attempt-a",
+  executionGeneration: 1,
+} as const;
+const context = { accountId: "account", workspaceId: "workspace", actor };
+let event: any;
+let existing: any;
+const policy = mock(async () => ({ defaultScope: "personal", subjectId: "user-a" }));
+const save = mock(async (_db: any, _context: any, request: any) => ({
+  entryId: request.entryId,
+  revisionId: "revision",
+  outcome: "pending",
+}));
+mock.module("@opengeni/db", () => ({
+  freezeAgentLearningPolicy: policy,
+  nestedPostgresSqlState: (error: any) => error.code,
+  getSessionTurn: async () => ({ sessionId: "session-a", triggerEventId: "message-a" }),
+  getSessionEvent: async () => event,
+  getKnowledgeEntry: async () => existing,
+  saveKnowledgeEntry: save,
+}));
+const { retainKnowledgeMessage } = await import("./knowledge-messages");
+afterAll(() => mock.restore());
+beforeEach(() => {
+  event = {
+    id: "message-a",
+    sessionId: "session-a",
+    type: "user.message",
+    payload: { text: "I confirm it is 21k.\nKeep this exact." },
+    occurredAt: "2026-09-11T12:00:00Z",
+  };
+  existing = null;
+  save.mockClear();
+});
+test("retains exact text with message identity and preserves the review receipt", async () => {
+  const result = await retainKnowledgeMessage({} as never, context);
+  expect(result).toMatchObject({ retained: true, outcome: "pending", messageId: "message-a" });
+  expect(save.mock.calls[0]?.[2].entry).toMatchObject({
+    content: event.payload.text,
+    source: {
+      kind: "conversation",
+      sessionId: "session-a",
+      externalId: "message-a",
+      capturedAt: event.occurredAt,
+    },
+  });
+  expect(save.mock.calls[0]?.[2].scope).toBeUndefined();
+});
+test("refuses foreign-session and non-user events without writing", async () => {
+  event.sessionId = "private-foreign";
+  expect(await retainKnowledgeMessage({} as never, context, "foreign-event")).toMatchObject({
+    retained: false,
+  });
+  event.sessionId = "session-a";
+  event.type = "assistant.message";
+  expect(await retainKnowledgeMessage({} as never, context)).toMatchObject({ retained: false });
+  expect(save).not.toHaveBeenCalled();
+});
+test("reuses a retained message without creating another review request", async () => {
+  existing = {
+    archived: false,
+    revision: {
+      id: "already-retained",
+      outcome: "pending",
+      entry: { content: event.payload.text },
+    },
+  };
+  expect(await retainKnowledgeMessage({} as never, context)).toMatchObject({
+    retained: true,
+    reused: true,
+    revisionId: "already-retained",
+    outcome: "pending",
+  });
+  expect(save).not.toHaveBeenCalled();
+});
+test("never revives rejected, archived, or edited retained content", async () => {
+  for (const revision of [
+    { outcome: "rejected", content: event.payload.text },
+    { outcome: "published", content: "edited" },
+  ]) {
+    existing = { archived: false, revision: { ...revision, entry: { content: revision.content } } };
+    expect(await retainKnowledgeMessage({} as never, context)).toMatchObject({ retained: false });
+  }
+  existing.archived = true;
+  expect(await retainKnowledgeMessage({} as never, context)).toMatchObject({ retained: false });
+  expect(save).not.toHaveBeenCalled();
+});

package/src/domain/knowledge-messages.ts

@@ -0,0 +1,106 @@
+import { createHash } from "node:crypto";
+import {
+  freezeAgentLearningPolicy,
+  getKnowledgeEntry,
+  getSessionEvent,
+  getSessionTurn,
+  nestedPostgresSqlState,
+  saveKnowledgeEntry,
+  type Database,
+  type KnowledgeContext,
+} from "@opengeni/db";
+
+function stableId(value: string) {
+  const hex = createHash("sha256").update(value).digest("hex");
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
+}
+
+/** Retain actual message bytes, never a model-authored reconstruction of a confirmation. */
+export async function retainKnowledgeMessage(
+  db: Database,
+  context: KnowledgeContext,
+  messageId?: string,
+) {
+  if (context.actor.kind !== "agent") throw new Error("An exact agent attempt is required");
+  // Validates the live attempt and resolves its frozen personal/shared authoring scope.
+  const policy = await freezeAgentLearningPolicy(db, context);
+  const turn = await getSessionTurn(db, context.workspaceId, context.actor.turnId);
+  if (!turn || turn.sessionId !== context.actor.sessionId) throw new Error("Message unavailable");
+  const eventId = messageId ?? turn.triggerEventId;
+  const event = eventId ? await getSessionEvent(db, context.workspaceId, eventId) : null;
+  const payload = event?.payload;
+  const text =
+    payload && typeof payload === "object" && "text" in payload ? payload.text : undefined;
+  if (
+    !event ||
+    event.sessionId !== context.actor.sessionId ||
+    event.type !== "user.message" ||
+    typeof text !== "string" ||
+    !text.trim()
+  ) {
+    return {
+      retained: false,
+      reason:
+        "Choose a user message from this conversation. The current turn may have been triggered by a tool or scheduled event.",
+    };
+  }
+  const key = `${context.accountId}:${context.workspaceId}:${policy.defaultScope}:${policy.subjectId ?? ""}:${event.id}`;
+  const entryId = stableId(`knowledge-message:${key}`);
+  for (const view of ["published", "needs_review"] as const) {
+    const existing = await getKnowledgeEntry(db, context, entryId, { view });
+    if (!existing) continue;
+    if (
+      existing.archived ||
+      existing.revision.outcome === "rejected" ||
+      existing.revision.entry.content !== text
+    ) {
+      return {
+        retained: false,
+        reason:
+          "This retained message was edited, archived or rejected. Inspect its history; it has not been overwritten or revived.",
+        entryId,
+      };
+    }
+    return {
+      retained: true,
+      entryId,
+      revisionId: existing.revision.id,
+      outcome: existing.revision.outcome,
+      messageId: event.id,
+      reused: true,
+    };
+  }
+  let receipt;
+  try {
+    receipt = await saveKnowledgeEntry(db, context, {
+      operationId: crypto.randomUUID(),
+      entryId,
+      expectedVersion: 0,
+      entry: {
+        kind: "source",
+        title: `Chat: ${text.replace(/\s+/g, " ").slice(0, 100)}`,
+        content: text,
+        source: {
+          kind: "conversation",
+          purpose: "evidence",
+          sessionId: event.sessionId,
+          externalId: event.id,
+          capturedAt: event.occurredAt,
+          retention: "full_text",
+        },
+      },
+    });
+  } catch (error) {
+    // An inaccessible archived/rejected source or a concurrent creation must not
+    // be overwritten. The stable entry ID makes retries converge without a new copy.
+    if (nestedPostgresSqlState(error) === "40001")
+      return {
+        retained: false,
+        entryId,
+        reason:
+          "This message already has a retained source or changed concurrently. Inspect available Knowledge before retrying; no source was overwritten or revived.",
+      };
+    throw error;
+  }
+  return { retained: true, ...receipt, messageId: event.id };
+}

package/src/domain/knowledge-preparation.test.ts

@@ -0,0 +1,276 @@
+import { expect, test } from "bun:test";
+import type { KnowledgeEntryRecord, KnowledgeEntrySummary } from "@opengeni/contracts";
+import { prepareKnowledgeSave } from "./knowledge-preparation";
+
+const context = {
+  accountId: "account",
+  workspaceId: "workspace",
+  actor: { kind: "human", review: true },
+} as never;
+const db = {} as never;
+const embedder = () => {
+  throw new Error("not used by fake search");
+};
+function group(title: string, parentIds: string[] = []): KnowledgeEntryRecord {
+  const id = crypto.randomUUID();
+  return {
+    id,
+    scope: "workspace",
+    version: 1,
+    publishedRevisionId: id,
+    latestRevisionId: id,
+    archived: false,
+    createdAt: "2026-09-14T00:00:00Z",
+    updatedAt: "2026-09-14T00:00:00Z",
+    revision: {
+      id,
+      entryId: id,
+      number: 1,
+      change: "upsert",
+      previousRevisionId: null,
+      restoredFromRevisionId: null,
+      createdAt: "2026-09-14T00:00:00Z",
+      createdBySessionId: null,
+      reviewBatchId: null,
+      outcome: "published",
+      entry: {
+        title,
+        kind: "group",
+        content: `Description of ${title}`,
+        groupIds: parentIds,
+        evidence: [],
+        relationships: [],
+      },
+    },
+  };
+}
+const summary = (record: KnowledgeEntryRecord) =>
+  ({
+    ...record,
+    revision: {
+      ...record.revision,
+      title: record.revision.entry.title,
+      kind: "group",
+      preview: "preview",
+      groupIds: record.revision.entry.groupIds,
+      sourceKind: null,
+    },
+    excerpts: [],
+  }) as KnowledgeEntrySummary;
+
+test("one read returns the complete paginated hierarchy and separately labeled duplicate candidates", async () => {
+  const parent = group("Engineering");
+  const child = group("Product UI", [parent.id]);
+  const pending = group("Proposed collection");
+  pending.revision.outcome = "pending";
+  const revoked = group("Revoked");
+  const calls: unknown[] = [];
+  const result = await prepareKnowledgeSave(
+    db,
+    context,
+    { query: "plugin setup consistency" },
+    embedder,
+    {
+      list: async (actualDb, actualContext, request) => {
+        expect(actualDb).toBe(db);
+        expect(actualContext).toBe(context);
+        calls.push(request);
+        if (request?.view === "needs_review")
+          return { entries: [summary(pending)], nextCursor: null };
+        return request?.cursor
+          ? { entries: [summary(child), summary(revoked)], nextCursor: null }
+          : { entries: [summary(parent)], nextCursor: "page2" };
+      },
+      get: async (_db, _context, id, options) => {
+        expect(options?.revisionId).toBeUndefined();
+        return [parent, child, pending].find((record) => record.id === id) ?? null;
+      },
+      search: async (_db, actualContext, request) => {
+        expect(actualContext).toBe(context);
+        expect(request.query).toBe("plugin setup consistency");
+        expect(request.groupId).toBeUndefined();
+        expect(request.scope).toBeUndefined();
+        return {
+          entries: [summary(request.view === "needs_review" ? pending : child)],
+          nextCursor: null,
+          searchMode: "hybrid",
+        };
+      },
+    },
+  );
+  expect(calls).toHaveLength(3);
+  expect(result.collections.complete).toBe(true);
+  expect(result.collections.entries.map((entry) => entry.title)).toEqual([
+    "Engineering",
+    "Product UI",
+    "Proposed collection",
+  ]);
+  expect(result.collections.entries[1]?.parentIds).toEqual([parent.id]);
+  expect(result.collections.entries[0]?.description).toBe("Description of Engineering");
+  expect(result.collections.entries[2]?.view).toBe("needs_review");
+  expect(result.matches.published.entries[0]?.id).toBe(child.id);
+  expect(result.matches.needs_review.entries[0]?.id).toBe(pending.id);
+});
+
+test("large collection maps explicitly continue and long descriptions report loss", async () => {
+  const record = group("Large collection");
+  record.revision.entry.content = "x".repeat(3000);
+  let count = 0;
+  const services: NonNullable<Parameters<typeof prepareKnowledgeSave>[4]> = {
+    list: async (_db, _context, request) => {
+      if (request?.view === "needs_review") return { entries: [], nextCursor: null };
+      return { entries: [summary(record)], nextCursor: `page${++count}` };
+    },
+    get: async () => record,
+    search: async () => ({ entries: [], nextCursor: null, searchMode: "keyword" }),
+  };
+  const result = await prepareKnowledgeSave(db, context, { query: "subject" }, embedder, services);
+  expect(result.collections.complete).toBe(false);
+  expect(result.collections.nextCursors.published).toBeTruthy();
+  expect(result.collections.nextCursors.needs_review).toBeNull();
+  expect(result.collections.entries[0]?.descriptionTruncated).toBe(true);
+  expect(result.collections.entries[0]?.description).toHaveLength(2000);
+  let requestedCursor: string | undefined;
+  await prepareKnowledgeSave(
+    db,
+    context,
+    { query: "subject", collectionCursors: result.collections.nextCursors },
+    embedder,
+    {
+      ...services,
+      list: async (_db, _context, request) => {
+        expect(request?.view).toBe("published");
+        requestedCursor = request?.cursor;
+        return { entries: [], nextCursor: null };
+      },
+    },
+  );
+  expect(requestedCursor).toBe(result.collections.nextCursors.published!);
+});
+
+test("a catalog authorization failure is not disguised as an empty complete map", async () => {
+  await expect(
+    prepareKnowledgeSave(db, context, { query: "subject" }, embedder, {
+      list: async () => {
+        throw new Error("access revoked");
+      },
+      get: async () => null,
+      search: async () => ({ entries: [], nextCursor: null, searchMode: "keyword" }),
+    }),
+  ).rejects.toThrow("access revoked");
+});
+
+test("read-only principals without proposal access fail before searching or embedding", async () => {
+  let touched = false;
+  await expect(
+    prepareKnowledgeSave(
+      db,
+      {
+        accountId: "account",
+        workspaceId: "workspace",
+        actor: { kind: "service", review: false },
+      } as never,
+      { query: "subject" },
+      () => {
+        touched = true;
+        throw new Error("must not embed");
+      },
+      {
+        list: async () => {
+          touched = true;
+          return { entries: [], nextCursor: null };
+        },
+        get: async () => {
+          touched = true;
+          return null;
+        },
+        search: async () => {
+          touched = true;
+          return { entries: [], nextCursor: null, searchMode: "keyword" };
+        },
+      },
+    ),
+  ).rejects.toMatchObject({ code: "42501" });
+  expect(touched).toBe(false);
+});
+
+test("published and pending searches reuse one query embedding", async () => {
+  let calls = 0;
+  await prepareKnowledgeSave(
+    db,
+    context,
+    { query: "plugin dialog" },
+    () => ({
+      model: "test",
+      dimensions: 2,
+      embedMany: async () => [],
+      embedQuery: async () => {
+        calls++;
+        return [1, 0];
+      },
+    }),
+    {
+      list: async () => ({ entries: [], nextCursor: null }),
+      get: async () => null,
+      search: async (_db, _context, request, queryEmbedder) => {
+        expect(await queryEmbedder().embedQuery(request.query!)).toEqual([1, 0]);
+        return { entries: [], nextCursor: null, searchMode: "hybrid" };
+      },
+    },
+  );
+  expect(calls).toBe(1);
+});
+
+test("a keyword fallback in the first view never retries an unfunded provider in the review view", async () => {
+  const searches: Array<{ mode: string | undefined; settingsProvided: boolean }> = [];
+  await prepareKnowledgeSave(
+    db,
+    context,
+    { query: "Acme renewal" },
+    () => {
+      throw new Error("An unfunded provider was called");
+    },
+    {
+      list: async () => ({ entries: [], nextCursor: null }),
+      get: async () => null,
+      search: async (_db, _context, request, _embedder, settings) => {
+        searches.push({ mode: request.mode, settingsProvided: !!settings });
+        return { entries: [], nextCursor: null, searchMode: "keyword" };
+      },
+    },
+    { documentEmbeddingProvider: "openai", documentEmbeddingBillingMode: "credits" } as never,
+  );
+  expect(searches).toEqual([
+    { mode: "keyword", settingsProvided: false },
+    { mode: "keyword", settingsProvided: false },
+  ]);
+});
+
+test("catalog rechecks current metadata after a collection changes", async () => {
+  const old = group("Old name");
+  const current = {
+    ...old,
+    version: 2,
+    revision: {
+      ...old.revision,
+      id: crypto.randomUUID(),
+      entry: { ...old.revision.entry, title: "New name" },
+    },
+  };
+  const result = await prepareKnowledgeSave(db, context, { query: "subject" }, embedder, {
+    list: async (_db, _context, request) => ({
+      entries: request?.view === "published" ? [summary(old)] : [],
+      nextCursor: null,
+    }),
+    get: async (_db, _context, _id, options) => {
+      expect(options?.revisionId).toBeUndefined();
+      return current;
+    },
+    search: async () => ({ entries: [], nextCursor: null, searchMode: "keyword" }),
+  });
+  expect(result.collections.entries[0]).toMatchObject({
+    title: "New name",
+    version: 2,
+    revisionId: current.revision.id,
+  });
+});

package/src/domain/knowledge-preparation.ts

@@ -0,0 +1,147 @@
+import {
+  KnowledgeSavePreparationRequest,
+  type KnowledgeSavePreparationResponse,
+} from "@opengeni/contracts";
+import {
+  getKnowledgeEntry,
+  listKnowledgeEntries,
+  type Database,
+  type KnowledgeContext,
+} from "@opengeni/db";
+import type { DocumentEmbedder } from "@opengeni/documents";
+import type { Settings } from "@opengeni/config";
+import { paidDocumentEmbedding } from "../billing/limits";
+import { searchKnowledgeEntries } from "./knowledge-search";
+
+const defaults = {
+  get: getKnowledgeEntry,
+  list: listKnowledgeEntries,
+  search: searchKnowledgeEntries,
+};
+// Normal workspaces return the whole map in one call. Larger maps explicitly
+// continue at a complete page boundary, never silently omit collections.
+const CATALOG_PAGE_SIZE = 25;
+const CATALOG_PAGE_BUDGET = 40;
+const DESCRIPTION_CHARS = 2000;
+const CATALOG_BYTES = 256 * 1024;
+
+export async function prepareKnowledgeSave(
+  db: Database,
+  context: KnowledgeContext,
+  input: KnowledgeSavePreparationRequest,
+  embedder: () => DocumentEmbedder,
+  services = defaults,
+  settings?: Settings,
+): Promise<KnowledgeSavePreparationResponse> {
+  const request = KnowledgeSavePreparationRequest.parse(input);
+  if (context.actor.kind !== "agent" && !(context.actor.kind === "human" && context.actor.review)) {
+    throw Object.assign(
+      new Error("Preparing Knowledge requires a live agent or human Knowledge reviewer"),
+      { code: "42501" },
+    );
+  }
+  let provider: DocumentEmbedder | undefined;
+  let queryEmbedding: Promise<number[]> | undefined;
+  const sharedEmbedder = (): DocumentEmbedder => {
+    provider ??= embedder();
+    const current = provider;
+    return {
+      model: current.model,
+      dimensions: current.dimensions,
+      embedMany: (texts) => current.embedMany(texts),
+      embedQuery: (query) => (queryEmbedding ??= current.embedQuery(query)),
+    };
+  };
+  const collections: KnowledgeSavePreparationResponse["collections"] = {
+    entries: [],
+    complete: true,
+    nextCursors: { published: null, needs_review: null },
+  };
+  // Search is independent of collection placement and always includes the
+  // separate unapproved view. No authoring policy or write is invoked here.
+  // Preparation returns BOTH views plus a catalog. A charge on the first
+  // semantic search would precede the second view and catalog; any later
+  // failure would leave no deliverable preparation result. Paid mode uses
+  // keyword discovery until a single-operation settlement seam exists.
+  const paid = settings != null && paidDocumentEmbedding(settings);
+  const published = await services.search(
+    db,
+    context,
+    {
+      query: request.query,
+      limit: request.limit,
+      view: "published",
+      ...(paid ? { mode: "keyword" as const } : {}),
+    },
+    sharedEmbedder,
+    paid ? undefined : settings,
+  );
+  // Reuse the first request's provider result (and charge). If it fell back
+  // to keyword because funding or the provider was unavailable, do not retry
+  // an unfunded embedding request in the second review view.
+  const needsReview = await services.search(
+    db,
+    context,
+    {
+      query: request.query,
+      limit: request.limit,
+      view: "needs_review",
+      ...(published.searchMode === "keyword" ? { mode: "keyword" as const } : {}),
+    },
+    sharedEmbedder,
+  );
+  const matches = { published, needs_review: needsReview };
+  for (const view of ["published", "needs_review"] as const) {
+    if (request.collectionCursors && request.collectionCursors[view] === null) continue;
+    let cursor = request.collectionCursors?.[view] ?? undefined;
+    let bytes = 0;
+    for (let pageNumber = 0; pageNumber < CATALOG_PAGE_BUDGET; pageNumber++) {
+      const page = await services.list(db, context, {
+        kind: "group",
+        view,
+        limit: CATALOG_PAGE_SIZE,
+        ...(cursor ? { cursor } : {}),
+      });
+      const records: Array<Awaited<ReturnType<typeof getKnowledgeEntry>>> = [];
+      for (let offset = 0; offset < page.entries.length; offset += 4) {
+        records.push(
+          ...(await Promise.all(
+            page.entries
+              .slice(offset, offset + 4)
+              .map((summary) => services.get(db, context, summary.id, { view })),
+          )),
+        );
+      }
+      for (const record of records) {
+        // Re-read the current view: an exact historical revision may remain
+        // readable after replacement and must not be called the current map.
+        if (
+          !record ||
+          record.archived ||
+          record.revision.entry.kind !== "group" ||
+          record.revision.outcome !== (view === "published" ? "published" : "pending")
+        )
+          continue;
+        const entry = record.revision.entry;
+        const descriptor = {
+          id: record.id,
+          revisionId: record.revision.id,
+          version: record.version,
+          scope: record.scope,
+          view,
+          title: entry.title,
+          description: entry.content.slice(0, DESCRIPTION_CHARS),
+          descriptionTruncated: entry.content.length > DESCRIPTION_CHARS,
+          parentIds: entry.groupIds,
+        };
+        collections.entries.push(descriptor);
+        bytes += Buffer.byteLength(JSON.stringify(descriptor), "utf8");
+      }
+      cursor = page.nextCursor ?? undefined;
+      if (!cursor || bytes >= CATALOG_BYTES) break;
+    }
+    collections.nextCursors[view] = cursor ?? null;
+    if (cursor) collections.complete = false;
+  }
+  return { collections, matches };
+}