@kelpie/server 0.1.0

package/src/modules/search/repository.ts

@@ -0,0 +1,388 @@
+import type { SearchCollection } from '@kelpie/schemas'
+import { and, asc, desc, eq, max, sql } from 'drizzle-orm'
+import type { SQL } from 'drizzle-orm'
+import { unionAll } from 'drizzle-orm/pg-core'
+import type { PgColumn } from 'drizzle-orm/pg-core'
+
+import type { Queryable } from '../../runtime/transaction.ts'
+import { companies } from '../companies/schema.ts'
+import { deals } from '../deals/schema.ts'
+import { decisions } from '../decisions/schema.ts'
+import { handbookPages } from '../handbook/schema.ts'
+import { roles } from '../hiring/schema.ts'
+import { opportunities } from '../opportunities/schema.ts'
+import { partnerships } from '../partnerships/schema.ts'
+import { people } from '../people/schema.ts'
+import { pipelineStages } from '../pipelines/schema.ts'
+import { planItems } from '../plans/schema.ts'
+import { positions } from '../positions/schema.ts'
+import { raises } from '../raises/schema.ts'
+
+/**
+ * The reads behind `GET /v1/search`: one query per collection, each against that
+ * collection's own `search_vector` GIN index.
+ *
+ * Eleven tables and no repository imports, the same licence `dashboard` takes and
+ * for the same reason. A search that composed nine repositories would ask each of
+ * them for a page it then had to re-rank, and none of them can rank at all.
+ *
+ * **Every branch of every query has to be indexable.** Four collections match on a
+ * second table as well: a Person is found by a job title that lives on Position,
+ * and a Deal, Opportunity or Raise by a step that lives on Plan. Written as
+ * `own_vector @@ q OR EXISTS (…)` that is a sequential scan, because Postgres will
+ * not use an index for an `OR` whose other branch is a subquery. Written as a
+ * `UNION ALL` of two matches, each branch uses its own index and the ranks are
+ * merged afterwards. That is why the shape below looks indirect.
+ */
+
+/** A row on its way to becoming a result. `subtitle` and `snippet` are per collection. */
+export interface SearchHit {
+  readonly id: string
+  readonly title: string
+  readonly subtitle: string | null
+  readonly snippetSource: string | null
+}
+
+/** What one collection answered: a capped, ranked page and the exact number of matches. */
+export interface CollectionHits {
+  readonly collection: SearchCollection
+  readonly hits: readonly SearchHit[]
+  readonly total: number
+}
+
+/**
+ * How far a match on a related record sits below a match on the record itself.
+ *
+ * A Deal whose own summary says "renewal" is a better answer than one with a
+ * planned step that mentions it. Without this they interleave by `ts_rank` alone,
+ * which compares the wrong two things: one rank is over the record, the other over
+ * a single line attached to it.
+ */
+const RELATED_RANK_FACTOR = 0.4
+
+/** The compiled `tsquery`, built once per request and shared by every collection. */
+export function compileQuery(tsQuery: string): SQL {
+  return sql`to_tsquery('english', ${tsQuery})`
+}
+
+function matches(vector: PgColumn, query: SQL): SQL {
+  return sql`${vector} @@ ${query}`
+}
+
+function rank(vector: PgColumn, query: SQL): SQL<number> {
+  return sql<number>`ts_rank(${vector}, ${query})`
+}
+
+function relatedRank(vector: PgColumn, query: SQL): SQL<number> {
+  return sql<number>`ts_rank(${vector}, ${query}) * ${RELATED_RANK_FACTOR}`
+}
+
+/**
+ * The number of matching records, which is not the number of rows returned.
+ *
+ * A window function is evaluated before `LIMIT`, so this counts every group the
+ * query found and the page below it stays capped. `api.md` has no envelope for
+ * this endpoint and the dashboard's precedent is an exact total beside a short
+ * list, because a capped list cannot answer "how many".
+ *
+ * Cast to `int` because `count` is `bigint`, and postgres.js hands a bigint back
+ * as a string rather than losing precision on it. Summing those gives `'01'`
+ * instead of `1`, which is a wrong number rather than a type error.
+ */
+const total = sql<number>`(count(*) over ())::int`
+
+/** Rows are grouped by id, so a record matching on both itself and a related row appears once. */
+function bestRank(rankColumn: PgColumn | SQL.Aliased<number>): SQL<number> {
+  return sql<number>`${max(rankColumn)}`
+}
+
+export async function searchHandbookPages(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const rows = await db
+    .select({
+      id: handbookPages.id,
+      title: handbookPages.title,
+      subtitle: handbookPages.slug,
+      snippetSource: handbookPages.body,
+      total,
+    })
+    .from(handbookPages)
+    .where(and(eq(handbookPages.workspaceId, workspaceId), matches(handbookPages.searchVector, query)))
+    .orderBy(desc(rank(handbookPages.searchVector, query)), asc(handbookPages.id))
+    .limit(limit)
+
+  return collect('handbook_page', rows)
+}
+
+export async function searchPeople(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const own = db
+    .select({ id: people.id, rank: rank(people.searchVector, query).as('rank') })
+    .from(people)
+    .where(and(eq(people.workspaceId, workspaceId), matches(people.searchVector, query)))
+
+  // A person is found by the title they hold, which is on Position and never on
+  // Person (`brief.md`). `?q=` on people already reaches the same way.
+  const byTitle = db
+    .select({ id: positions.personId, rank: relatedRank(positions.searchVector, query).as('rank') })
+    .from(positions)
+    .where(and(eq(positions.workspaceId, workspaceId), matches(positions.searchVector, query)))
+
+  const hits = unionAll(own, byTitle).as('hits')
+
+  const rows = await db
+    .select({
+      id: people.id,
+      title: people.name,
+      subtitle: people.email,
+      snippetSource: people.summary,
+      total,
+    })
+    .from(hits)
+    .innerJoin(people, eq(people.id, hits.id))
+    .groupBy(people.id)
+    .orderBy(desc(bestRank(hits.rank)), asc(people.id))
+    .limit(limit)
+
+  return collect('person', rows)
+}
+
+export async function searchRoles(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const rows = await db
+    .select({
+      id: roles.id,
+      title: roles.title,
+      subtitle: roles.status,
+      // A Role is a title and a status. There is no prose on it to excerpt.
+      snippetSource: sql<string | null>`null::text`,
+      total,
+    })
+    .from(roles)
+    .where(and(eq(roles.workspaceId, workspaceId), matches(roles.searchVector, query)))
+    .orderBy(desc(rank(roles.searchVector, query)), asc(roles.id))
+    .limit(limit)
+
+  return collect('role', rows)
+}
+
+export async function searchCompanies(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const rows = await db
+    .select({
+      id: companies.id,
+      title: companies.name,
+      subtitle: companies.domain,
+      snippetSource: companies.summary,
+      total,
+    })
+    .from(companies)
+    .where(and(eq(companies.workspaceId, workspaceId), matches(companies.searchVector, query)))
+    .orderBy(desc(rank(companies.searchVector, query)), asc(companies.id))
+    .limit(limit)
+
+  return collect('company', rows)
+}
+
+/**
+ * A Deal, Opportunity or Raise found by one of its Plan items.
+ *
+ * `plan_items` is polymorphic with no foreign key, so the target type is part of
+ * the match rather than something a join expresses.
+ */
+function byPlanItem(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  targetType: 'deal' | 'opportunity' | 'raise',
+) {
+  return db
+    .select({ id: planItems.targetId, rank: relatedRank(planItems.searchVector, query).as('rank') })
+    .from(planItems)
+    .where(
+      and(
+        eq(planItems.workspaceId, workspaceId),
+        eq(planItems.targetType, targetType),
+        matches(planItems.searchVector, query),
+      ),
+    )
+}
+
+export async function searchDeals(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const own = db
+    .select({ id: deals.id, rank: rank(deals.searchVector, query).as('rank') })
+    .from(deals)
+    .where(and(eq(deals.workspaceId, workspaceId), matches(deals.searchVector, query)))
+
+  const hits = unionAll(own, byPlanItem(db, workspaceId, query, 'deal')).as('hits')
+
+  const rows = await db
+    .select({
+      id: deals.id,
+      title: deals.name,
+      subtitle: pipelineStages.label,
+      snippetSource: deals.summary,
+      total,
+    })
+    .from(hits)
+    .innerJoin(deals, eq(deals.id, hits.id))
+    .innerJoin(pipelineStages, eq(pipelineStages.id, deals.stageId))
+    .groupBy(deals.id, pipelineStages.label)
+    .orderBy(desc(bestRank(hits.rank)), asc(deals.id))
+    .limit(limit)
+
+  return collect('deal', rows)
+}
+
+export async function searchOpportunities(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const own = db
+    .select({ id: opportunities.id, rank: rank(opportunities.searchVector, query).as('rank') })
+    .from(opportunities)
+    .where(and(eq(opportunities.workspaceId, workspaceId), matches(opportunities.searchVector, query)))
+
+  const hits = unionAll(own, byPlanItem(db, workspaceId, query, 'opportunity')).as('hits')
+
+  const rows = await db
+    .select({
+      id: opportunities.id,
+      title: opportunities.name,
+      subtitle: opportunities.kind,
+      snippetSource: opportunities.summary,
+      total,
+    })
+    .from(hits)
+    .innerJoin(opportunities, eq(opportunities.id, hits.id))
+    .groupBy(opportunities.id)
+    .orderBy(desc(bestRank(hits.rank)), asc(opportunities.id))
+    .limit(limit)
+
+  return collect('opportunity', rows)
+}
+
+export async function searchRaises(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const own = db
+    .select({ id: raises.id, rank: rank(raises.searchVector, query).as('rank') })
+    .from(raises)
+    .where(and(eq(raises.workspaceId, workspaceId), matches(raises.searchVector, query)))
+
+  const hits = unionAll(own, byPlanItem(db, workspaceId, query, 'raise')).as('hits')
+
+  const rows = await db
+    .select({
+      id: raises.id,
+      title: raises.name,
+      subtitle: pipelineStages.label,
+      snippetSource: raises.summary,
+      total,
+    })
+    .from(hits)
+    .innerJoin(raises, eq(raises.id, hits.id))
+    .innerJoin(pipelineStages, eq(pipelineStages.id, raises.stageId))
+    .groupBy(raises.id, pipelineStages.label)
+    .orderBy(desc(bestRank(hits.rank)), asc(raises.id))
+    .limit(limit)
+
+  return collect('raise', rows)
+}
+
+export async function searchPartnerships(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const rows = await db
+    .select({
+      id: partnerships.id,
+      title: partnerships.name,
+      subtitle: partnerships.kind,
+      snippetSource: partnerships.summary,
+      total,
+    })
+    .from(partnerships)
+    .where(and(eq(partnerships.workspaceId, workspaceId), matches(partnerships.searchVector, query)))
+    .orderBy(desc(rank(partnerships.searchVector, query)), asc(partnerships.id))
+    .limit(limit)
+
+  return collect('partnership', rows)
+}
+
+export async function searchDecisions(
+  db: Queryable,
+  workspaceId: string,
+  query: SQL,
+  limit: number,
+): Promise<CollectionHits> {
+  const rows = await db
+    .select({
+      id: decisions.id,
+      title: decisions.body,
+      // The date it was decided, as `YYYY-MM-DD`. A decision has no name and no
+      // stage; when it was made is the line the mockup shows beside it.
+      subtitle: sql<string>`to_char(${decisions.decidedAt} at time zone 'UTC', 'YYYY-MM-DD')`,
+      snippetSource: decisions.rationale,
+      total,
+    })
+    .from(decisions)
+    .where(and(eq(decisions.workspaceId, workspaceId), matches(decisions.searchVector, query)))
+    .orderBy(desc(rank(decisions.searchVector, query)), asc(decisions.id))
+    .limit(limit)
+
+  return collect('decision', rows)
+}
+
+interface HitRow {
+  readonly id: string
+  readonly title: string
+  readonly subtitle: string | null
+  readonly snippetSource: string | null
+  readonly total: number
+}
+
+/**
+ * `count(*) over ()` is on every row, so an empty page carries no total at all.
+ * That is the only case where zero is the right answer.
+ */
+function collect(collection: SearchCollection, rows: readonly HitRow[]): CollectionHits {
+  return {
+    collection,
+    hits: rows.map(({ id, title, subtitle, snippetSource }) => ({
+      id,
+      title,
+      subtitle,
+      snippetSource,
+    })),
+    total: rows[0]?.total ?? 0,
+  }
+}

package/src/modules/search/routes.ts

@@ -0,0 +1,131 @@
+import { SEARCH_COLLECTIONS } from '@kelpie/schemas'
+import type { SearchCollection } from '@kelpie/schemas'
+import type { Context, Hono } from 'hono'
+
+import { AppError } from '../../lib/errors.ts'
+import { readPageSize } from '../../lib/pagination.ts'
+import type { Actor } from '../auth/actor.ts'
+import { resolveActorFrom } from '../auth/credentials.ts'
+import type { CredentialDependencies } from '../auth/credentials.ts'
+import { MAX_SEARCH_TERM_LENGTH } from './query.ts'
+import type { SearchGroup, SearchItem, SearchResults, SearchService } from './service.ts'
+
+/**
+ * Wire shape for `GET /v1/search`.
+ *
+ * One resource made of nine lists, so it takes the dashboard's shape rather than
+ * a list's: a bare object, one `?limit=` capping every group, and an exact total
+ * beside each capped group. There is no cursor. Paging nine keysets at once is a
+ * different feature, and the answer to "too many results" is a better query.
+ */
+
+export interface SearchRoutesDependencies extends CredentialDependencies {
+  readonly service: SearchService
+}
+
+/**
+ * `?q=`, required and never blank.
+ *
+ * `api.md` makes a blank filter value a 422 rather than something to ignore,
+ * because an empty search and a search for everything are different questions and
+ * only one of them was asked.
+ */
+function readTerm(context: Context): string {
+  const raw = context.req.query('q')
+
+  if (raw === undefined || raw.trim().length === 0) {
+    throw AppError.validationFailed('"q" is required', [
+      { field: 'q', message: 'Give something to search for' },
+    ])
+  }
+
+  if (raw.length > MAX_SEARCH_TERM_LENGTH) {
+    throw AppError.validationFailed('"q" is too long', [
+      { field: 'q', message: `Use at most ${String(MAX_SEARCH_TERM_LENGTH)} characters` },
+    ])
+  }
+
+  return raw
+}
+
+const COLLECTIONS = new Set<string>(SEARCH_COLLECTIONS)
+
+/**
+ * `?type=`, repeatable in the same way an id filter is (`api.md`): naming it twice
+ * asks for either. Absent means every collection.
+ *
+ * An unknown value is a 422 rather than an empty group. Silently searching eight
+ * collections when nine were named is the kind of answer a caller cannot tell from
+ * a genuine miss.
+ */
+function readCollections(context: Context): readonly SearchCollection[] | undefined {
+  const values = context.req.queries('type')
+
+  if (values === undefined || values.length === 0) {
+    return undefined
+  }
+
+  const unknown = values.filter((value) => !COLLECTIONS.has(value))
+
+  if (unknown.length > 0) {
+    throw AppError.validationFailed('"type" names a collection that does not exist', [
+      { field: 'type', message: `Use one of: ${SEARCH_COLLECTIONS.join(', ')}` },
+    ])
+  }
+
+  // Deduplicated, and put back into the canonical order so the groups come back
+  // the same way whatever order they were asked for in.
+  const named = new Set(values)
+
+  return SEARCH_COLLECTIONS.filter((collection) => named.has(collection))
+}
+
+function readLimit(context: Context): number | undefined {
+  const raw = context.req.query('limit')
+
+  return raw === undefined ? undefined : readPageSize(raw)
+}
+
+function itemBody(item: SearchItem): Record<string, unknown> {
+  return {
+    id: item.id,
+    title: item.title,
+    subtitle: item.subtitle,
+    snippet: item.snippet,
+  }
+}
+
+function groupBody(group: SearchGroup): Record<string, unknown> {
+  return {
+    type: group.collection,
+    total: group.total,
+    items: group.items.map(itemBody),
+  }
+}
+
+export function searchResponse(results: SearchResults): Record<string, unknown> {
+  return {
+    query: results.query,
+    limit: results.limit,
+    total: results.total,
+    groups: results.groups.map(groupBody),
+  }
+}
+
+export function mountSearchRoutes(router: Hono, dependencies: SearchRoutesDependencies): void {
+  const requireActor = (context: Context): Promise<Actor> => resolveActorFrom(dependencies, context)
+
+  router.get('/search', async (context) => {
+    // Credentials first. A caller with none should be told that, not told their
+    // query string is malformed.
+    const actor = await requireActor(context)
+
+    const results = await dependencies.service.find(actor, {
+      term: readTerm(context),
+      limit: readLimit(context),
+      collections: readCollections(context),
+    })
+
+    return context.json(searchResponse(results))
+  })
+}

package/src/modules/search/service.ts

@@ -0,0 +1,153 @@
+import { SEARCH_COLLECTIONS } from '@kelpie/schemas'
+import type { SearchCollection } from '@kelpie/schemas'
+
+import type { Database } from '../../lib/database.ts'
+import type { Actor } from '../auth/actor.ts'
+import { requireWorkspaceId } from '../auth/actor.ts'
+import { searchWords, snippet, toTsQuery } from './query.ts'
+import {
+  compileQuery,
+  searchCompanies,
+  searchDeals,
+  searchDecisions,
+  searchHandbookPages,
+  searchOpportunities,
+  searchPartnerships,
+  searchPeople,
+  searchRaises,
+  searchRoles,
+} from './repository.ts'
+import type { CollectionHits } from './repository.ts'
+
+/**
+ * One box across nine collections, ranked per collection rather than merged.
+ *
+ * A single merged ranking would need `ts_rank` values from different tables to
+ * mean the same thing, and they do not: a handbook page is a thousand words and a
+ * Role is three. Grouping is also what the reader wants, because "which Deal" and
+ * "which handbook page" are different questions.
+ */
+
+export interface SearchItem {
+  readonly id: string
+  readonly title: string
+  /** The record's one line of context. What it holds depends on the collection. */
+  readonly subtitle: string | null
+  /** A fragment of the record's prose, centred on the match. Empty when it has none. */
+  readonly snippet: string
+}
+
+export interface SearchGroup {
+  readonly collection: SearchCollection
+  /** Every match, not just the ones in `items`. */
+  readonly total: number
+  readonly items: readonly SearchItem[]
+}
+
+export interface SearchResults {
+  readonly query: string
+  readonly limit: number
+  /** Across every group. The number a "12 results" heading is built from. */
+  readonly total: number
+  /** Always all nine, in `SEARCH_COLLECTIONS` order, empty ones included. */
+  readonly groups: readonly SearchGroup[]
+}
+
+export interface SearchRequest {
+  readonly term: string
+  readonly limit?: number | undefined
+  /** Which collections to look through. Absent means all of them. */
+  readonly collections?: readonly SearchCollection[] | undefined
+}
+
+export interface SearchService {
+  find(actor: Actor, request: SearchRequest): Promise<SearchResults>
+}
+
+export interface SearchServiceDependencies {
+  readonly db: Database
+}
+
+/**
+ * How many rows a group carries when the caller does not say.
+ *
+ * Small on purpose: nine groups on one page, and a reader scanning them wants the
+ * best few of each rather than a page of People they have to scroll past to reach
+ * the handbook. The exact total sits beside each group regardless.
+ */
+export const DEFAULT_SEARCH_LIMIT = 10
+
+type CollectionSearch = (
+  db: Database,
+  workspaceId: string,
+  query: ReturnType<typeof compileQuery>,
+  limit: number,
+) => Promise<CollectionHits>
+
+const SEARCHES: Readonly<Record<SearchCollection, CollectionSearch>> = {
+  handbook_page: searchHandbookPages,
+  person: searchPeople,
+  role: searchRoles,
+  company: searchCompanies,
+  deal: searchDeals,
+  opportunity: searchOpportunities,
+  raise: searchRaises,
+  partnership: searchPartnerships,
+  decision: searchDecisions,
+}
+
+/** A term that tokenises to nothing still answers, with every group empty. */
+function emptyResults(term: string, limit: number, wanted: readonly SearchCollection[]): SearchResults {
+  return {
+    query: term,
+    limit,
+    total: 0,
+    groups: wanted.map((collection) => ({ collection, total: 0, items: [] })),
+  }
+}
+
+function toGroup(hits: CollectionHits, words: readonly string[]): SearchGroup {
+  return {
+    collection: hits.collection,
+    total: hits.total,
+    items: hits.hits.map((hit) => ({
+      id: hit.id,
+      title: hit.title,
+      subtitle: hit.subtitle,
+      snippet: hit.snippetSource === null ? '' : snippet(hit.snippetSource, words),
+    })),
+  }
+}
+
+export function createSearchService(dependencies: SearchServiceDependencies): SearchService {
+  return {
+    async find(actor, request) {
+      const workspaceId = requireWorkspaceId(actor)
+      const limit = request.limit ?? DEFAULT_SEARCH_LIMIT
+      const wanted = request.collections ?? SEARCH_COLLECTIONS
+      const tsQuery = toTsQuery(request.term)
+
+      if (tsQuery === null) {
+        return emptyResults(request.term, limit, wanted)
+      }
+
+      const query = compileQuery(tsQuery)
+      const words = searchWords(request.term)
+
+      // Nine independent reads with nothing to share but the compiled query. Run
+      // in sequence they would be nine round trips deep rather than wide.
+      const groups = await Promise.all(
+        wanted.map(async (collection) =>
+          toGroup(await SEARCHES[collection](dependencies.db, workspaceId, query, limit), words),
+        ),
+      )
+
+      return {
+        query: request.term,
+        limit,
+        total: groups.reduce((sum, group) => sum + group.total, 0),
+        groups,
+      }
+    },
+  }
+}