@pepys/api 0.0.1 → 0.1.0

package/src/queries/neighbourhood/walk-neighbourhood.ts

@@ -0,0 +1,121 @@
+import type { CoreOutputs, CoreParsedInputs } from '@pepys/contract';
+import { neighbourhoodCaps } from '@pepys/contract';
+import { parseEntityID } from '@pepys/kit';
+import type { Result } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import { compareText } from '../../utils/compare-text.ts';
+import type { QueryContext, QueryFailure } from '../types.ts';
+import { pickLinkTypes } from './pick-link-types.ts';
+import { takeHop } from './take-hop.ts';
+import type { WalkState } from './take-hop.ts';
+
+type NeighbourhoodInput = CoreParsedInputs['neighbourhood'];
+
+type NeighbourhoodOutput = CoreOutputs['neighbourhood'];
+
+// A breadth-first walk with one read of the link view per hop. Every consumer with a current
+// version is attached, and none is required, so a consumer with none is skipped and named.
+export async function walkNeighbourhood(
+  context: QueryContext,
+  input: NeighbourhoodInput,
+): Promise<Result<NeighbourhoodOutput, QueryFailure<'NOT_FOUND' | 'BAD_REQUEST'>>> {
+  const parsed = parseEntityID(input.id);
+
+  if (!parsed.ok) {
+    const message = `${input.id} is not an entity ID: ${parsed.failure.reason}`;
+
+    return { ok: false, failure: { code: 'BAD_REQUEST', message } };
+  }
+
+  const kind = `${parsed.value.source}:${parsed.value.kind}`;
+
+  if (!context.deployment.entityKinds.some((known) => known === kind)) {
+    return {
+      ok: false,
+      failure: { code: 'NOT_FOUND', message: `no plug-in brings the kind ${kind}` },
+    };
+  }
+
+  const types = pickLinkTypes(context.linkRegistry, input);
+
+  if (!types.ok) {
+    const message = `unknown link types ${types.failure.unknown.join(', ')}; the known ones are ${types.failure.known.join(', ')}`;
+
+    return { ok: false, failure: { code: 'BAD_REQUEST', message } };
+  }
+
+  const read = await context.store.withConsumers(
+    [],
+    async (db, call) => {
+      let state: WalkState = {
+        depths: new Map([[input.id, 0]]),
+        links: [],
+        frontier: [input.id],
+        isTruncated: false,
+      };
+
+      for (let depth = 1; depth <= input.depthLimit && state.frontier.length > 0; depth += 1) {
+        const frontier = state.frontier;
+
+        const rows =
+          types.value.length === 0
+            ? []
+            : await db
+                .selectFrom('link_effective')
+                .selectAll()
+                .where('type', 'in', types.value)
+                .where('confidence', '>=', input.minimumConfidence)
+                .where((builder) =>
+                  builder.or([
+                    builder('from_id', 'in', frontier),
+                    builder('to_id', 'in', frontier),
+                  ]),
+                )
+                .orderBy('type')
+                .orderBy('from_id')
+                .orderBy('to_id')
+                .orderBy('origin')
+                .limit(neighbourhoodCaps.linkReadLimit + 1)
+                .execute();
+
+        state = takeHop(state, rows, {
+          depth,
+          entityLimit: input.entityLimit,
+          linkReadLimit: neighbourhoodCaps.linkReadLimit,
+        });
+      }
+
+      const entities = await db
+        .selectFrom('entity')
+        .selectAll()
+        .where('id', 'in', [...state.depths.keys()])
+        .execute();
+
+      return { state, entities, skipped: call.skipped };
+    },
+    { whenCurrent: context.deployment.consumers },
+  );
+
+  invariant(read.ok, 'a call with no required consumer always runs');
+
+  const state = read.value.state;
+
+  const found = new Set(read.value.entities.map((entity) => entity.id));
+
+  const entities = read.value.entities.map((entity) =>
+    Object.assign(entity, { depth: state.depths.get(entity.id) ?? 0 }),
+  );
+
+  return {
+    ok: true,
+    value: {
+      entities: entities.toSorted(
+        (first, second) => first.depth - second.depth || compareText(first.id, second.id),
+      ),
+      links: state.links.toSorted((first, second) => first.depth - second.depth),
+      unresolved: [...state.depths.keys()].filter((id) => !found.has(id)).toSorted(),
+      skippedConsumers: [...read.value.skipped],
+      isTruncated: state.isTruncated,
+    },
+  };
+}

package/src/queries/types.ts

@@ -1,12 +1,19 @@
-import type { ConsumerDeclaration } from '@pepys/kit';
-import type { StoreReader } from '../store-reader/store-reader.ts';
+import type { CheckedDeployment, LinkRegistry, StoreReader } from '@pepys/kit';
+import type { EntityKindIndex } from './entity-kinds/build-entity-kind-index.ts';
 
 export interface QueryContext {
   store: StoreReader;
-  consumers: readonly ConsumerDeclaration[];
+  deployment: CheckedDeployment;
+  entityKinds: EntityKindIndex;
+  linkRegistry: LinkRegistry;
 }
 
-export interface QueryFailure {
-  code: 'NOT_FOUND';
+export interface QueryFailure<
+  Code extends 'NOT_FOUND' | 'BAD_REQUEST' | 'SERVICE_UNAVAILABLE' =
+    | 'NOT_FOUND'
+    | 'BAD_REQUEST'
+    | 'SERVICE_UNAVAILABLE',
+> {
+  code: Code;
   message: string;
 }

package/src/serve/serve-api.ts

@@ -0,0 +1,53 @@
+import { checkDeployment } from '@pepys/kit';
+import type { Deployment } from '@pepys/kit';
+import { makeShutdownHandler } from '@pepys/schema';
+import { createApp } from '../app/create-app.ts';
+import { readAPIConfig } from '../config/read-api-config.ts';
+import { buildQueryContext } from '../queries/build-query-context.ts';
+import { createStoreReader } from '../store-reader/create-store-reader.ts';
+
+interface ServeAPIOptions {
+  deployment: Deployment;
+  environment: Readonly<Record<string, string | undefined>>;
+}
+
+// The API process of a deployment: oRPC at /rpc and MCP at /mcp, on the listen address from the
+// environment. It exits 1 at start on a failed deployment check or a bad variable.
+export function serveAPI(options: ServeAPIOptions): void {
+  const checked = checkDeployment(options.deployment);
+
+  if (!checked.ok) {
+    console.error('the deployment fails its check', JSON.stringify(checked.failure));
+    process.exit(1);
+  }
+
+  const config = readAPIConfig(options.environment);
+
+  if (!config.ok) {
+    console.error(config.failure.join('\n'));
+    process.exit(1);
+  }
+
+  const store = createStoreReader({ storeDirectory: config.value.storeDirectory });
+  const app = createApp(buildQueryContext(store, checked.value));
+
+  app.listen({ hostname: config.value.host, port: config.value.port });
+  console.log(`api listening on ${config.value.host}:${app.server?.port}`);
+
+  const shutdown = makeShutdownHandler({
+    stop: async () => {
+      await app.stop();
+      await store.close();
+    },
+    logError: (message, error) => {
+      console.error(message, error);
+    },
+    exit: (code) => process.exit(code),
+  });
+
+  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.on(signal, () => {
+      void shutdown();
+    });
+  }
+}

package/src/store-reader/build-shared-views-plugin.ts

@@ -0,0 +1,86 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import {
+  DummyDriver,
+  Kysely,
+  SelectQueryNode,
+  SqliteAdapter,
+  SqliteIntrospector,
+  SqliteQueryCompiler,
+  sql,
+} from 'kysely';
+import type { KyselyPlugin, WithNode } from 'kysely';
+import invariant from 'tiny-invariant';
+
+// Turso refuses a TEMP view over an attached file, so the shared views are common table expressions
+// in front of every query. Each file holds its own `entity` view, and a bare name reads the first
+// attached file's alone; a common table expression of the same name wins over it.
+export function buildSharedViewsPlugin(attached: readonly ConsumerDeclaration[]): KyselyPlugin {
+  const views = collectSharedViews(attached);
+
+  return {
+    transformQuery: (args) => {
+      if (!SelectQueryNode.is(args.node)) {
+        return args.node;
+      }
+
+      return {
+        ...args.node,
+        with: {
+          ...views,
+          expressions: [...views.expressions, ...(args.node.with?.expressions ?? [])],
+        },
+      };
+    },
+    transformResult: (args) => Promise.resolve(args.result),
+  };
+}
+
+// A builder that compiles and never connects: it only makes the operation nodes. Its tables are
+// the attached files' own, named `<consumer>.entity` and `<consumer>.link`.
+const builder = new Kysely<Record<string, Record<string, unknown>>>({
+  dialect: {
+    createAdapter: () => new SqliteAdapter(),
+    createDriver: () => new DummyDriver(),
+    createIntrospector: (db) => new SqliteIntrospector(db),
+    createQueryCompiler: () => new SqliteQueryCompiler(),
+  },
+});
+
+const entityColumns = ['id', 'kind', 'title', 'url', 'actor', 'updated_at'];
+const linkColumns = ['type', 'from_id', 'to_id', 'scope', 'origin', 'confidence', 'evidence'];
+
+function collectSharedViews(attached: readonly ConsumerDeclaration[]): WithNode {
+  const entityFiles = attached.filter((consumer) => consumer.entityViews.length > 0);
+  const linkFiles = attached.filter((consumer) => consumer.links.length > 0);
+
+  const withNode = builder
+    .with('entity', (creator) => {
+      const [first, ...rest] = entityFiles.map((consumer) =>
+        creator.selectFrom(`${consumer.name}.entity`).select(entityColumns),
+      );
+
+      // no attached file has an entity view: the same columns, and no rows
+      if (first === undefined) {
+        return creator.selectNoFrom(entityColumns.map((column) => sql`null`.as(column))).limit(0);
+      }
+
+      return rest.reduce((union, next) => union.unionAll(next), first);
+    })
+    .with('link_effective', (creator) => {
+      const [first, ...rest] = linkFiles.map((consumer) =>
+        creator.selectFrom(`${consumer.name}.link`).select(linkColumns),
+      );
+
+      if (first === undefined) {
+        return creator.selectNoFrom(linkColumns.map((column) => sql`null`.as(column))).limit(0);
+      }
+
+      return rest.reduce((union, next) => union.unionAll(next), first);
+    })
+    .selectNoFrom(sql`1`.as('one'))
+    .toOperationNode().with;
+
+  invariant(withNode !== undefined, 'a query with two common table expressions has a with node');
+
+  return withNode;
+}

package/src/store-reader/create-store-reader.ts

@@ -1,116 +1,173 @@
 import { realpath } from 'node:fs/promises';
-import { join } from 'node:path';
-import type { Result } from '@pepys/kit';
+import { basename, dirname, join } from 'node:path';
+import type { ConsumerDeclaration, ConsumersDB, StoreReader } from '@pepys/kit';
 import { createTursoDialect } from '@pepys/kysely-turso';
+import { connect } from '@tursodatabase/database';
 import type { Database } from '@tursodatabase/database';
 import { Kysely } from 'kysely';
-import type { Transaction } from 'kysely';
-import type { ConsumerDB, StoreReader } from './store-reader.ts';
-
-interface OpenConsumer {
-  target: string;
-  database: Kysely<ConsumerDB>;
-  readers: number;
-  isRetired: boolean;
-}
+import { buildSharedViewsPlugin } from './build-shared-views-plugin.ts';
+import { pickAttachedFiles } from './pick-attached-files.ts';
+import type { AttachedFile, CurrentFile } from './pick-attached-files.ts';
 
 interface StoreReaderOptions {
   storeDirectory: string;
-  openDatabase: (path: string) => Promise<Database>;
 }
 
-export function createStoreReader(options: StoreReaderOptions): StoreReader {
-  const open = new Map<string, OpenConsumer>();
-  const retired = new Set<OpenConsumer>();
-
-  const withConsumer = async <Value>(
-    consumerName: string,
-    work: (transaction: Transaction<ConsumerDB>) => Promise<Value>,
-  ): Promise<Result<Value, { reason: 'no-current-version' }>> => {
-    // the link swings to a new version directory when `pepys up` swaps, and the old file stays
-    const target = await readCurrentTarget(join(options.storeDirectory, consumerName, 'current'));
-
-    if (target === null) {
-      return { ok: false, failure: { reason: 'no-current-version' } };
-    }
-
-    const opened = openConsumer(consumerName, target);
-    const consumer = opened.consumer;
-
-    consumer.readers += 1;
-
-    if (opened.replaced !== undefined) {
-      await destroyIfDrained(opened.replaced);
-    }
+// One open read-only handle on a consumer file. Turso 0.8.3 refuses to attach a file that the
+// host holds open with its multi-process WAL, unless this process holds a handle of its own.
+interface FileKeeper {
+  path: string;
+  database: Promise<Database>;
+  readers: number;
+  isRetired: boolean;
+}
 
-    try {
-      // one transaction keeps every read of a call on one connection and one snapshot
-      return { ok: true, value: await consumer.database.transaction().execute(work) };
-    } finally {
-      consumer.readers -= 1;
+// The Turso adapter of the store reader port. Each call opens one connection on an empty
+// in-memory database, attaches the current file of each consumer read-only under the consumer's
+// name, and runs the work in one transaction, which holds one snapshot of each file.
+export function createStoreReader(options: StoreReaderOptions): StoreReader {
+  const keepers = new Map<string, FileKeeper>();
+  const retired = new Set<FileKeeper>();
 
-      await destroyIfDrained(consumer);
-    }
-  };
+  const holdKeeper = (file: AttachedFile): FileKeeper => {
+    const held = keepers.get(file.consumer.name);
 
-  const openConsumer = (
-    consumerName: string,
-    target: string,
-  ): { consumer: OpenConsumer; replaced?: OpenConsumer } => {
-    const existing = open.get(consumerName);
+    if (held?.path === file.path) {
+      held.readers += 1;
 
-    if (existing?.target === target) {
-      return { consumer: existing };
+      return held;
     }
 
-    const consumer: OpenConsumer = {
-      target,
-      database: new Kysely<ConsumerDB>({
-        dialect: createTursoDialect({
-          database: () => options.openDatabase(join(target, 'consumer.db')),
-        }),
-      }),
-      readers: 0,
+    const keeper: FileKeeper = {
+      path: file.path,
+      database: connect(file.path, { readonly: true, experimental: ['multiprocess_wal'] }),
+      readers: 1,
       isRetired: false,
     };
 
-    open.set(consumerName, consumer);
+    keepers.set(file.consumer.name, keeper);
 
-    if (existing === undefined) {
-      return { consumer };
+    // the host swapped the version: the old file closes once its last call ends
+    if (held !== undefined) {
+      held.isRetired = true;
+
+      retired.add(held);
     }
 
-    existing.isRetired = true;
+    return keeper;
+  };
 
-    retired.add(existing);
+  const releaseKeeper = async (keeper: FileKeeper): Promise<void> => {
+    keeper.readers -= 1;
 
-    return { consumer, replaced: existing };
-  };
+    if (keeper.isRetired && keeper.readers === 0 && retired.delete(keeper)) {
+      const database = await keeper.database;
 
-  const destroyIfDrained = async (consumer: OpenConsumer): Promise<void> => {
-    if (consumer.isRetired && consumer.readers === 0 && retired.delete(consumer)) {
-      await consumer.database.destroy();
+      await database.close();
     }
   };
 
-  const close = async (): Promise<void> => {
-    const consumers = [...open.values(), ...retired];
-
-    open.clear();
-    retired.clear();
-
-    // a destroy waits for the reads still running on the database
-    await Promise.all(consumers.map((consumer) => consumer.database.destroy()));
+  return {
+    withConsumers: async (consumers, work, callOptions) => {
+      // every file resolves before anything opens, so a swap during the call changes nothing
+      const [required, optional] = await Promise.all([
+        readCurrentFiles(options.storeDirectory, consumers),
+        readCurrentFiles(options.storeDirectory, callOptions?.whenCurrent ?? []),
+      ]);
+
+      const plan = pickAttachedFiles(required, optional);
+
+      if (plan.kind === 'missing') {
+        return { ok: false, failure: { reason: 'no-current-version', consumers: plan.consumers } };
+      }
+
+      const held = plan.files.map((file) => holdKeeper(file));
+
+      try {
+        await Promise.all(held.map((keeper) => keeper.database));
+
+        const connection = await connect(':memory:', { experimental: ['attach'] });
+
+        const db = new Kysely<ConsumersDB<typeof consumers>>({
+          dialect: createTursoDialect({ database: () => Promise.resolve(connection) }),
+        });
+
+        try {
+          for (const file of plan.files) {
+            await connection.exec(formatAttachStatement(file));
+          }
+
+          const call = {
+            attached: Object.fromEntries(
+              plan.files.map((file): [string, { version: string }] => [
+                file.consumer.name,
+                { version: basename(dirname(file.path)) },
+              ]),
+            ),
+            skipped: plan.skipped,
+          };
+
+          const views = buildSharedViewsPlugin(plan.files.map((file) => file.consumer));
+
+          // A failed write ends Turso's transaction itself, and Kysely's own rollback would then
+          // fail and hide the first error, so the call opens and ends the transaction by hand.
+          await connection.exec('begin');
+
+          try {
+            const value = await work(db.withPlugin(views), call);
+
+            return { ok: true, value };
+          } finally {
+            if (connection.inTransaction) {
+              await connection.exec('rollback');
+            }
+          }
+        } finally {
+          await db.destroy();
+        }
+      } finally {
+        await Promise.all(held.map((keeper) => releaseKeeper(keeper)));
+      }
+    },
+    close: async () => {
+      const open = [...keepers.values(), ...retired];
+
+      keepers.clear();
+      retired.clear();
+
+      await Promise.all(
+        open.map(async (keeper) => {
+          const database = await keeper.database;
+
+          await database.close();
+        }),
+      );
+    },
   };
+}
 
-  return { withConsumer, close };
+function readCurrentFiles(
+  storeDirectory: string,
+  consumers: readonly ConsumerDeclaration[],
+): Promise<CurrentFile[]> {
+  return Promise.all(
+    consumers.map(async (consumer) => {
+      const path = await readRealPath(
+        join(storeDirectory, consumer.name, 'current', 'consumer.db'),
+      );
+
+      return { consumer, path };
+    }),
+  );
 }
 
-async function readCurrentTarget(currentLink: string): Promise<string | null> {
+async function readRealPath(path: string): Promise<string | null> {
   try {
-    return await realpath(currentLink);
+    const resolved = await realpath(path);
+
+    return resolved;
   } catch (error: unknown) {
-    if (isMissingFileError(error)) {
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
       return null;
     }
 
@@ -118,6 +175,13 @@ async function readCurrentTarget(currentLink: string): Promise<string | null> {
   }
 }
 
-function isMissingFileError(error: unknown): boolean {
-  return error instanceof Error && 'code' in error && error.code === 'ENOENT';
+// A URI opens the file read-only. The path is percent-encoded, so a `?`, `#`, `%` or quote in a
+// folder name stays part of the path.
+function formatAttachStatement(file: AttachedFile): string {
+  const encoded = encodeURI(file.path)
+    .replaceAll('?', '%3F')
+    .replaceAll('#', '%23')
+    .replaceAll("'", '%27');
+
+  return `ATTACH 'file:${encoded}?mode=ro' AS "${file.consumer.name.replaceAll('"', '""')}"`;
 }

package/src/store-reader/pick-attached-files.ts

@@ -0,0 +1,45 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+
+export interface CurrentFile {
+  consumer: ConsumerDeclaration;
+
+  // the consumer file of the current version, every link resolved; null with no current version
+  path: string | null;
+}
+
+export interface AttachedFile {
+  consumer: ConsumerDeclaration;
+  path: string;
+}
+
+export type AttachPlan =
+  | { kind: 'missing'; consumers: readonly string[] }
+  | { kind: 'attach'; files: readonly AttachedFile[]; skipped: readonly string[] };
+
+// Which files one call attaches: every required consumer's, else the call fails naming each one
+// with no current version; then each optional consumer's that has one. A name attaches once.
+export function pickAttachedFiles(
+  required: readonly CurrentFile[],
+  optional: readonly CurrentFile[],
+): AttachPlan {
+  const missing = required.filter((file) => file.path === null).map((file) => file.consumer.name);
+
+  if (missing.length > 0) {
+    return { kind: 'missing', consumers: missing };
+  }
+
+  const files = new Map<string, AttachedFile>();
+  const skipped = new Set<string>();
+
+  for (const file of [...required, ...optional]) {
+    const name = file.consumer.name;
+
+    if (file.path === null) {
+      skipped.add(name);
+    } else if (!files.has(name)) {
+      files.set(name, { consumer: file.consumer, path: file.path });
+    }
+  }
+
+  return { kind: 'attach', files: [...files.values()], skipped: [...skipped] };
+}

package/src/store-reader/read-change-operations.ts

@@ -1,29 +1,38 @@
-import type { Result } from '@pepys/kit';
+import type { ConsumerDeclaration, Result, StoreReader, StoreReaderFailure } from '@pepys/kit';
+import { sql } from 'kysely';
 import { z } from 'zod';
-import type { StoreReader } from './store-reader.ts';
 
 interface ChangeOperation {
   key: string;
   operation: string;
 }
 
+const changeLogRowSchema = z.object({ changes: z.string() });
+const changeLogRowsSchema = z.object({ rows: z.array(changeLogRowSchema) });
+
+const changeSchema = z.object({
+  key: z.string(),
+  headers: z.object({ operation: z.string() }),
+});
+
 // each change log row holds one batch as a JSON list of State Protocol messages
-const headersSchema = z.object({ operation: z.string() });
-const changesSchema = z.array(z.object({ key: z.string(), headers: headersSchema }));
+const changesSchema = z.array(changeSchema);
 
 export async function readChangeOperations(
   store: StoreReader,
-  consumerName: string,
-): Promise<Result<ChangeOperation[], { reason: 'no-current-version' }>> {
-  const result = await store.withConsumer(consumerName, (transaction) =>
-    transaction.selectFrom('change_log').select('changes').orderBy('batch').execute(),
+  consumer: ConsumerDeclaration,
+): Promise<Result<ChangeOperation[], StoreReaderFailure>> {
+  const table = sql.table(`${consumer.name}.change_log`);
+
+  const result = await store.withConsumers([consumer], (db) =>
+    sql`select changes from ${table} order by batch`.execute(db),
   );
 
   if (!result.ok) {
     return result;
   }
 
-  const operations = result.value.flatMap((row) => {
+  const operations = changeLogRowsSchema.parse(result.value).rows.flatMap((row) => {
     const changes: unknown = JSON.parse(row.changes);
 
     return changesSchema