@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/org/graph.ts

@@ -0,0 +1,318 @@
+// brain org: an item's entities and relations, kept WITHOUT a leak, plus the per-(user, project) principals that make project-aware
+// ranking possible. Everything here is server-side and used by server.ts; none of it changes src/org/types.ts.
+//
+// ENTITIES AND RELATIONS
+//   Chitta's entity graph is shared by everybody in the store (names fold onto one canonical entity across records), so feeding it
+//   from private items would be a side channel. The server therefore never gives Chitta a graph (`extractGraph: false`). Entities
+//   and relations live in two tables of the organisation database, keyed by the item (`rid`):
+//     org_item_entities  (rid, ekey, name, type)             ekey = the normalised name (see normKey)
+//     org_item_relations (rid, from_key, from_name, to_key, to_name, type)
+//   They have no meaning without their item row: a trigger on org_items (db.ts) deletes them with the item, whatever removes it
+//   (forget, retention, an admin). They carry no visibility of their own: EVERY read joins `org_items` and applies the same
+//   predicate recall uses (db.VISIBLE_SQL) in the same statement, so promotion, demotion, team and membership changes apply at
+//   once, and a row of an item the caller may not see is not counted, ranked, named or typed. Display names and types are taken
+//   from VISIBLE rows only (an entity spelled "ACME Corp" in a private item and "Acme Corp" in a team item shows the team spelling
+//   to everybody else).
+//
+// WHAT USES THEM
+//   * recall: after the direct (Chitta-ranked) hits, other visible items of the same project that share an entity (or a relation
+//     endpoint) with the top hits are pulled in, ranked by how many entities they share, with a score below every direct hit. They
+//     fill free slots of `limit` and the token budget, never more (see server.ts).
+//   * GET /v1/graph?project=<id>[&entity=<name>]  (a new endpoint; the existing shapes are untouched)
+//       without `entity`:  {project, entities: [{name, type?, items}]}            the most-used entities of the project (at most 100)
+//                          `items` counts the VISIBLE items that mention the entity (as entity or relation endpoint)
+//       with `entity`:     {project, entity: {name, type?, items} | null,
+//                           neighbours: [{name, relation, direction: "out"|"in", items}],   (at most 100)
+//                           items: [{id, name, kind, scope, owner, team?, source}]}         (visible items that mention it, newest first, at most 50)
+//       An entity that does not exist, and one that only exists in items the caller may not see, give the SAME answer
+//       (`entity: null`, empty lists). Item text is not part of this answer: use recall for text.
+//
+// PROJECT-AWARE RANKING (principals, all names made of ids that contain no ':')
+//   h = first 32 hex of sha256(project id)
+//   sy:<user>:<h>     the SYNTHETIC searching principal of one user in one project. Chitta ranks only what this principal may see.
+//   pj:<h>            members of the project (the synthetic principal belongs to it only while the user is a member)
+//   tm:<h>:<team>     a team's items in the project (belongs to it for each team the user is in)
+//   og:<h>            org-wide items in the project (every synthetic principal belongs to it)
+//   An item's permission edges: the owner (`u:<owner>`, kept so the owner can always delete it, and `sy:<owner>:<h>`), plus the
+//   group of its scope. The synthetic principal's `belongsTo` edges are diffed against the user's CURRENT memberships at every
+//   recall (a few small queries), so a membership or team change applies on the next call and nothing is rebuilt in bulk.
+import type { Database } from "bun:sqlite"
+import { sha256Hex } from "./auth.ts"
+import { VISIBLE_SQL, type ItemRow, type OrgDb } from "./db.ts"
+import { redact } from "../memory/redact.ts"
+import type { Scope } from "./types.ts"
+
+// ---- sanitising what a client sends ----------------------------------------------------------------------------------------
+
+export interface Entity {
+  name: string
+  type?: string
+}
+export interface Relation {
+  from: string
+  to: string
+  type: string
+}
+export interface Extras {
+  entities: Entity[]
+  relations: Relation[]
+}
+
+export const MAX_ENTITY_NAME = 200
+const MAX_ENTITIES = 100
+const MAX_RELATIONS = 100
+
+/** The comparison key of an entity name: Unicode-normalised, lower-cased, whitespace collapsed. "" means unusable. */
+export function normKey(name: string): string {
+  return name.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim().slice(0, MAX_ENTITY_NAME)
+}
+
+// eslint-disable-next-line no-control-regex
+const CONTROL = /[\u0000-\u001f\u007f]/
+const cleanName = (x: unknown): string | undefined => {
+  if (typeof x !== "string") return undefined
+  const s = redact(x).replace(/\s+/g, " ").trim()
+  // a name that held a secret says nothing useful and would link unrelated items together
+  if (!s || s.length > MAX_ENTITY_NAME || CONTROL.test(s) || s.includes("[redacted]")) return undefined
+  return normKey(s) ? s : undefined
+}
+const cleanType = (x: unknown): string | undefined => {
+  if (typeof x !== "string") return undefined
+  const s = x.trim().toLowerCase()
+  return /^[a-z0-9][a-z0-9_ -]{0,39}$/.test(s) ? s : undefined
+}
+const cleanPredicate = (x: unknown): string | undefined => {
+  if (typeof x !== "string") return undefined
+  const s = x.trim().toLowerCase().replace(/[^a-z0-9_]+/g, "_").replace(/^_+|_+$/g, "").slice(0, 60)
+  return s || undefined
+}
+
+/** The usable, redacted entities and relations of a request item. Malformed entries are dropped, never an error (they were always
+ *  accepted and ignored before), and the lists are capped. */
+export function cleanExtras(entities: unknown, relations: unknown): Extras {
+  const es = new Map<string, Entity>()
+  if (Array.isArray(entities)) {
+    for (const e of entities) {
+      if (es.size >= MAX_ENTITIES) break
+      if (typeof e !== "object" || e === null) continue
+      const name = cleanName((e as Record<string, unknown>).name)
+      if (!name) continue
+      const type = cleanType((e as Record<string, unknown>).type)
+      const k = normKey(name)
+      if (!es.has(k)) es.set(k, { name, ...(type ? { type } : {}) })
+    }
+  }
+  const rs = new Map<string, Relation>()
+  if (Array.isArray(relations)) {
+    for (const r of relations) {
+      if (rs.size >= MAX_RELATIONS) break
+      if (typeof r !== "object" || r === null) continue
+      const o = r as Record<string, unknown>
+      const from = cleanName(o.from)
+      const to = cleanName(o.to)
+      const type = cleanPredicate(o.type)
+      if (!from || !to || !type) continue
+      rs.set(`${normKey(from)}\u0000${type}\u0000${normKey(to)}`, { from, to, type })
+    }
+  }
+  return { entities: [...es.values()], relations: [...rs.values()] }
+}
+
+// ---- principals ------------------------------------------------------------------------------------------------------------
+
+export const projectHash = (project: string): string => sha256Hex(project).slice(0, 32)
+export const synthPrincipal = (user: string, project: string): string => `sy:${user}:${projectHash(project)}`
+const groupOf = {
+  project: (h: string) => `pj:${h}`,
+  team: (h: string, team: string) => `tm:${h}:${team}`,
+  org: (h: string) => `og:${h}`,
+}
+
+/** Who gets a permission edge to an item: the owner (two principals) and the group of its scope. */
+export function aclPrincipals(row: Pick<ItemRow, "owner" | "project" | "scope" | "team">): string[] {
+  const h = projectHash(row.project)
+  const out = [`u:${row.owner}`, `sy:${row.owner}:${h}`]
+  if (row.scope === "project") out.push(groupOf.project(h))
+  else if (row.scope === "team" && row.team) out.push(groupOf.team(h, row.team))
+  else if (row.scope === "org") out.push(groupOf.org(h))
+  return out
+}
+
+interface StoreLike {
+  db: Database
+  addEdge(src: string, dst: string, label: string): void
+  bumpVersion(): void
+}
+
+const marks = (n: number): string => Array.from({ length: n }, () => "?").join(",")
+
+export class OrgGraph {
+  constructor(
+    private readonly org: OrgDb,
+    private readonly store: StoreLike,
+    /** Registers a principal as a Chitta user (Chitta's search looks the searching principal up). */
+    private readonly registerPrincipal: (id: string) => void = () => {},
+  ) {}
+  private readonly registered = new Set<string>()
+
+  private get db(): Database {
+    return this.org.db
+  }
+
+  // ---- project-aware ranking -----------------------------------------------------------------------------------------------
+
+  /** Makes the synthetic principal of (user, project) belong to exactly the groups the user's CURRENT memberships give.
+   *  Synchronous; returns the principal to search as. */
+  syncSynthetic(userId: string, project: string): string {
+    const h = projectHash(project)
+    const s = `sy:${userId}:${h}`
+    if (!this.registered.has(s)) (this.registerPrincipal(s), this.registered.add(s))
+    const want = new Set<string>([groupOf.org(h)])
+    if (this.org.isProjectMember(userId, project)) want.add(groupOf.project(h))
+    for (const t of this.org.teamsOf(userId)) want.add(groupOf.team(h, t))
+    const have = new Set((this.store.db.query("SELECT dst FROM edges WHERE src = ? AND label = 'belongsTo'").all(s) as Array<{ dst: string }>).map((r) => r.dst).filter((d) => /^(?:pj|tm|og):/.test(d)))
+    let changed = false
+    for (const g of want) if (!have.has(g)) (this.store.addEdge(s, g, "belongsTo"), (changed = true))
+    for (const g of have) {
+      if (want.has(g)) continue
+      this.store.db.query("DELETE FROM edges WHERE src = ? AND dst = ? AND label = 'belongsTo'").run(s, g)
+      changed = true
+    }
+    if (changed) this.store.bumpVersion()
+    return s
+  }
+
+  /** True while the database still carries the pre-1.1 (global team/project/anyone group) ACL. */
+  aclIsLegacy(): boolean {
+    return (this.db.query("SELECT value FROM org_meta WHERE key = 'acl_scheme'").get() as { value: string } | null)?.value !== "2"
+  }
+  markAclCurrent(): void {
+    this.db.query("INSERT INTO org_meta (key, value) VALUES ('acl_scheme', '2') ON CONFLICT(key) DO UPDATE SET value = excluded.value").run()
+  }
+  /** Removes what the old scheme left in Chitta's tables: the global `team:`/`proj:` group nodes and edges. */
+  dropLegacyGroups(): void {
+    const q = (sql: string) => this.store.db.query(sql).run()
+    q("DELETE FROM edges WHERE dst LIKE 'team:%' OR dst LIKE 'proj:%' OR src LIKE 'team:%' OR src LIKE 'proj:%'")
+    q("DELETE FROM nodes WHERE coll = 'groups' AND (id LIKE 'team:%' OR id LIKE 'proj:%')")
+    this.store.bumpVersion()
+  }
+  allItemRows(): ItemRow[] {
+    return this.db.query("SELECT * FROM org_items").all() as ItemRow[]
+  }
+
+  // ---- storing entities and relations ---------------------------------------------------------------------------------------
+
+  /** Replaces the item's entities and relations with these (an item stands for what its latest ingest said). */
+  saveExtras(rid: string, x: Extras): void {
+    this.org.tx(() => {
+      this.db.query("DELETE FROM org_item_entities WHERE rid = ?").run(rid)
+      this.db.query("DELETE FROM org_item_relations WHERE rid = ?").run(rid)
+      for (const e of x.entities) this.db.query("INSERT OR IGNORE INTO org_item_entities (rid, ekey, name, type) VALUES (?,?,?,?)").run(rid, normKey(e.name), e.name, e.type ?? null)
+      for (const r of x.relations) {
+        this.db.query("INSERT OR IGNORE INTO org_item_relations (rid, from_key, from_name, to_key, to_name, type) VALUES (?,?,?,?,?,?)").run(rid, normKey(r.from), r.from, normKey(r.to), r.to, r.type)
+      }
+    })
+  }
+  /** What an item stored (for re-checking the policy on promote). Callers must already have the right to this row. */
+  loadExtras(rid: string): Extras {
+    const entities = (this.db.query("SELECT name, type FROM org_item_entities WHERE rid = ? ORDER BY ekey").all(rid) as Array<{ name: string; type: string | null }>).map((e) => ({ name: e.name, ...(e.type ? { type: e.type } : {}) }))
+    const relations = (this.db.query("SELECT from_name AS \"from\", to_name AS \"to\", type FROM org_item_relations WHERE rid = ? ORDER BY from_key, type, to_key").all(rid) as Relation[])
+    return { entities, relations }
+  }
+
+  // ---- recall expansion ----------------------------------------------------------------------------------------------------
+
+  /**
+   * Other items of `project` that `user` may see and that share an entity or relation endpoint with the items `tops` (which the
+   * caller already may see), best overlap first. `exclude` are rids already in the answer. At most `max` rows.
+   * The visibility predicate is part of the SAME statement that counts the overlap, so an invisible item contributes nothing.
+   */
+  expansion(user: string, project: string, tops: ItemRow[], exclude: Set<string>, max: number): Array<{ rid: string; overlap: number }> {
+    if (max <= 0 || !tops.length) return []
+    const topIds = tops.map((t) => t.rid)
+    const skip = new Set(tops.map((t) => normKey(t.label)).concat(normKey(project)))
+    const keys = new Set<string>()
+    const add = (k: string) => k && !skip.has(k) && keys.add(k)
+    for (const r of this.db.query(`SELECT ekey FROM org_item_entities WHERE rid IN (${marks(topIds.length)})`).all(...(topIds as never[])) as Array<{ ekey: string }>) add(r.ekey)
+    for (const r of this.db.query(`SELECT from_key f, to_key t FROM org_item_relations WHERE rid IN (${marks(topIds.length)})`).all(...(topIds as never[])) as Array<{ f: string; t: string }>) (add(r.f), add(r.t))
+    if (!keys.size) return []
+    const kl = [...keys].slice(0, 200)
+    const params: Record<string, string | number> = { $u: user, $p: project, $n: max }
+    const kn = kl.map((k, i) => ((params[`$k${i}`] = k), `$k${i}`)).join(",")
+    const ex = [...new Set([...exclude, ...topIds])]
+    const en = ex.map((r, i) => ((params[`$x${i}`] = r), `$x${i}`)).join(",")
+    const sql = `SELECT org_items.rid AS rid, count(DISTINCT x.k) AS overlap FROM (
+        SELECT rid, ekey AS k FROM org_item_entities WHERE ekey IN (${kn})
+        UNION ALL SELECT rid, from_key FROM org_item_relations WHERE from_key IN (${kn})
+        UNION ALL SELECT rid, to_key FROM org_item_relations WHERE to_key IN (${kn})
+      ) x JOIN org_items ON org_items.rid = x.rid
+      WHERE org_items.project = $p AND ${ex.length ? `org_items.rid NOT IN (${en}) AND ` : ""}${VISIBLE_SQL}
+      GROUP BY org_items.rid ORDER BY overlap DESC, org_items.stored_at DESC, org_items.rid LIMIT $n`
+    return this.db.query(sql).all(params as never) as Array<{ rid: string; overlap: number }>
+  }
+
+  // ---- GET /v1/graph -------------------------------------------------------------------------------------------------------
+
+  view(user: string, project: string, entity?: string): GraphView | GraphEntityView {
+    const p = { $u: user, $p: project }
+    if (entity === undefined) {
+      const rows = this.db
+        .query(
+          `SELECT ekey, count(DISTINCT org_item_entities.rid) AS items, min(org_item_entities.name) AS name, min(org_item_entities.type) AS type
+           FROM org_item_entities JOIN org_items ON org_items.rid = org_item_entities.rid
+           WHERE org_items.project = $p AND ${VISIBLE_SQL} GROUP BY ekey ORDER BY items DESC, ekey LIMIT 100`,
+        )
+        .all(p as never) as Array<{ items: number; name: string; type: string | null }>
+      return { project, entities: rows.map((r) => ({ name: redact(r.name), ...(r.type ? { type: r.type } : {}), items: r.items })) }
+    }
+    const key = normKey(entity)
+    const kp = { ...p, $k: key }
+    const mention = `SELECT rid, name, type FROM org_item_entities WHERE ekey = $k
+      UNION ALL SELECT rid, from_name, NULL FROM org_item_relations WHERE from_key = $k
+      UNION ALL SELECT rid, to_name, NULL FROM org_item_relations WHERE to_key = $k`
+    const head = key
+      ? (this.db
+          .query(`SELECT count(DISTINCT m.rid) AS items, min(m.name) AS name, min(m.type) AS type FROM (${mention}) m JOIN org_items ON org_items.rid = m.rid WHERE org_items.project = $p AND ${VISIBLE_SQL}`)
+          .get(kp as never) as { items: number; name: string | null; type: string | null })
+      : { items: 0, name: null, type: null }
+    if (!head.items || head.name === null) return { project, entity: null, neighbours: [], items: [] }
+    const nb = this.db
+      .query(
+        `SELECT n.other_name AS name, n.type AS relation, n.dir AS direction, count(DISTINCT n.rid) AS items FROM (
+           SELECT rid, to_name AS other_name, to_key AS other_key, type, 'out' AS dir FROM org_item_relations WHERE from_key = $k
+           UNION ALL SELECT rid, from_name, from_key, type, 'in' FROM org_item_relations WHERE to_key = $k) n
+         JOIN org_items ON org_items.rid = n.rid WHERE org_items.project = $p AND ${VISIBLE_SQL}
+         GROUP BY n.other_key, n.type, n.dir ORDER BY items DESC, n.other_key, n.type LIMIT 100`,
+      )
+      .all(kp as never) as Array<{ name: string; relation: string; direction: "out" | "in"; items: number }>
+    const items = this.db
+      .query(
+        `SELECT DISTINCT org_items.item_id, org_items.name, org_items.kind, org_items.scope, org_items.owner, org_items.team, org_items.source, org_items.stored_at, org_items.rid
+         FROM (${mention}) m JOIN org_items ON org_items.rid = m.rid WHERE org_items.project = $p AND ${VISIBLE_SQL}
+         ORDER BY org_items.stored_at DESC, org_items.rid LIMIT 50`,
+      )
+      .all(kp as never) as Array<{ item_id: string; name: string; kind: string; scope: Scope; owner: string; team: string | null; source: string }>
+    return {
+      project,
+      entity: { name: redact(head.name), ...(head.type ? { type: head.type } : {}), items: head.items },
+      neighbours: nb.map((r) => ({ name: redact(r.name), relation: r.relation, direction: r.direction, items: r.items })),
+      items: items.map((r) => ({ id: r.item_id, name: r.name, kind: r.kind, scope: r.scope, owner: r.owner, ...(r.team ? { team: r.team } : {}), source: r.source })),
+    }
+  }
+}
+
+export interface GraphView {
+  project: string
+  entities: Array<{ name: string; type?: string; items: number }>
+}
+export interface GraphEntityView {
+  project: string
+  entity: { name: string; type?: string; items: number } | null
+  neighbours: Array<{ name: string; relation: string; direction: "out" | "in"; items: number }>
+  items: Array<{ id: string; name: string; kind: string; scope: Scope; owner: string; team?: string; source: string }>
+}
+
+/** The score of a graph-expanded item: below every direct hit (half the lowest), higher with more shared entities. */
+export function expansionScore(lowestDirect: number, overlap: number): number {
+  return lowestDirect * 0.5 * (overlap / (overlap + 1))
+}