@pepys/api 0.0.0-stage → 0.0.1

package/README.md

@@ -1,3 +1,31 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/api</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/api"><img src="https://img.shields.io/npm/v/@pepys/api" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a>
+  </p>
+</div>
+
+The pepys API, for Bun: typed queries over the store that the host writes.
+
+A deployment serves the core queries and each plug-in's queries over oRPC.
+
+## Install
+
+```sh
+bun add @pepys/api
+```
+
+## Features
+
+- The oRPC router and the app that serves it.
+- The store reader over the host's Turso files.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the API and the store.

package/package.json

@@ -1,6 +1,43 @@
 {
   "name": "@pepys/api",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.0.1",
+  "description": "The pepys API: typed queries over the store",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/api"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@orpc/contract": "1.16.1",
+    "@orpc/server": "1.16.1",
+    "@pepys/contract": "0.0.1",
+    "@pepys/kit": "0.0.1",
+    "@pepys/kysely-turso": "0.0.1",
+    "@pepys/schema": "0.0.1",
+    "@tursodatabase/database": "0.8.3",
+    "elysia": "2.0.0-rc.1",
+    "exact-mirror": "1.2.7",
+    "kysely": "0.29.8",
+    "openapi-types": "12.1.3",
+    "tiny-invariant": "1.3.3",
+    "ts-pattern": "5.9.2",
+    "typebox": "1.4.4",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@orpc/client": "1.16.1",
+    "@pepys/ingest": "0.0.1",
+    "@zgeoff/std": "0.4.3"
+  }
+}

package/src/app/create-app.ts

@@ -0,0 +1,14 @@
+import { RPCHandler } from '@orpc/server/fetch';
+import { Elysia } from 'elysia';
+import type { QueryContext } from '../queries/types.ts';
+import { createRouter } from './create-router.ts';
+
+export function createApp(context: QueryContext) {
+  const handler = new RPCHandler(createRouter());
+
+  return new Elysia().all('/rpc/*', { parse: 'none' }, async (route) => {
+    const result = await handler.handle(route.request, { prefix: '/rpc', context });
+
+    return result.matched ? result.response : new Response('not found', { status: 404 });
+  });
+}

package/src/app/create-router.ts

@@ -0,0 +1,68 @@
+import type { ContractRouterClient } from '@orpc/contract';
+import { ORPCError, ValidationError, implement } from '@orpc/server';
+import { coreContract } from '@pepys/contract';
+import type { CoreContract } from '@pepys/contract';
+import { readEntity } from '../queries/get-entity/read-entity.ts';
+import type { QueryContext } from '../queries/types.ts';
+
+export function createRouter() {
+  // oRPC validates the input before a procedure's own middleware by default, so a middleware
+  // could not turn an input failure into the procedure's typed error.
+  const api = implement(coreContract, {
+    initialInputValidationIndex: Number.POSITIVE_INFINITY,
+  }).$context<QueryContext>();
+
+  return api.router({
+    get_entity: api.get_entity.handler(async (options) => {
+      const result = await readEntity(options.context, options.input.id);
+
+      const buildNotFound = options.errors.NOT_FOUND;
+
+      if (!result.ok) {
+        throw buildNotFound({ message: result.failure.message });
+      }
+
+      return result.value;
+    }),
+    neighbourhood: api.neighbourhood.handler(throwNotImplemented),
+    search: api.search.handler(throwNotImplemented),
+    entity_history: api.entity_history.handler(throwNotImplemented),
+    assert_link: api.assert_link
+      .use(async (options) => {
+        const buildUnprocessable = options.errors.UNPROCESSABLE_CONTENT;
+
+        try {
+          return await options.next();
+        } catch (error) {
+          if (isMissingEvidence(error)) {
+            throw buildUnprocessable({ cause: error });
+          }
+
+          throw error;
+        }
+      })
+      .handler(throwNotImplemented),
+    retract_link: api.retract_link.handler(throwNotImplemented),
+    correct_label: api.correct_label.handler(throwNotImplemented),
+  });
+}
+
+export type APIClient = ContractRouterClient<CoreContract>;
+
+interface NotImplementedCall {
+  errors: { NOT_IMPLEMENTED: () => Error };
+}
+
+function throwNotImplemented(options: NotImplementedCall): never {
+  const buildNotImplemented = options.errors.NOT_IMPLEMENTED;
+
+  throw buildNotImplemented();
+}
+
+function isMissingEvidence(error: unknown): boolean {
+  return (
+    error instanceof ORPCError &&
+    error.cause instanceof ValidationError &&
+    error.cause.issues.some((issue) => issue.path?.[0] === 'evidence')
+  );
+}

package/src/change-feed/change-feed.ts

@@ -0,0 +1,33 @@
+import type { StreamOffset } from '@pepys/kit';
+
+export type ChangeMessage =
+  | {
+      kind: 'change';
+      type: string;
+      key: string;
+      value: unknown;
+      oldValue: unknown;
+      operation: 'insert' | 'update' | 'delete';
+    }
+  | { kind: 'snapshot-start' }
+  | { kind: 'snapshot-end' };
+
+export interface FeedStream {
+  consumer: string;
+  generation: number;
+}
+
+export type FeedPosition = StreamOffset | -1 | 'now';
+
+export type FeedRead =
+  | { kind: 'messages'; messages: readonly ChangeMessage[]; next: StreamOffset; atTail: boolean }
+  | { kind: 'closed'; currentGeneration: number }
+  | { kind: 'gone'; snapshotAt: StreamOffset };
+
+export interface ChangeFeed {
+  read: (
+    stream: FeedStream,
+    from: FeedPosition,
+    mode: 'catch-up' | 'long-poll',
+  ) => Promise<FeedRead>;
+}

package/src/config/read-api-config.ts

@@ -0,0 +1,26 @@
+import type { Result } from '@pepys/kit';
+import { parsePort } from '@pepys/schema';
+
+export interface APIConfig {
+  storeDirectory: string;
+  port: number;
+}
+
+export function readAPIConfig(
+  environment: Readonly<Record<string, string | undefined>>,
+): Result<APIConfig, string[]> {
+  const storeDirectory = environment['PEPYS_STORE_DIR'] ?? '';
+  const portText = environment['PEPYS_API_PORT'] ?? '';
+  const port = portText === '' ? ({ ok: true, value: 8081 } as const) : parsePort(portText);
+
+  const problems = [
+    ...(storeDirectory === '' ? ['PEPYS_STORE_DIR is not set'] : []),
+    ...(port.ok ? [] : ['PEPYS_API_PORT is not a port number']),
+  ];
+
+  if (!port.ok || problems.length > 0) {
+    return { ok: false, failure: problems };
+  }
+
+  return { ok: true, value: { storeDirectory, port: port.value } };
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+export { createRouter } from './app/create-router.ts';
+export type { APIClient } from './app/create-router.ts';
+export { createApp } from './app/create-app.ts';
+export type * from './change-feed/change-feed.ts';
+export type * from './principal/principal.ts';
+export { readAPIConfig } from './config/read-api-config.ts';
+export { createStoreReader } from './store-reader/create-store-reader.ts';
+export { openConsumerDatabase } from './store-reader/open-consumer-database.ts';
+export { readChangeOperations } from './store-reader/read-change-operations.ts';
+export { readOffsets } from './store-reader/read-offsets.ts';
+export { readPullRequests } from './store-reader/read-pull-requests.ts';

package/src/principal/principal.ts

@@ -0,0 +1,10 @@
+import type { ClientID, GrantID } from '@pepys/kit';
+
+export type Scope = 'read' | 'write';
+
+export interface Principal {
+  client: ClientID;
+  clientName: string;
+  grant: GrantID;
+  scopes: readonly Scope[];
+}

package/src/queries/get-entity/build-entity-row.ts

@@ -0,0 +1,23 @@
+import type { ColumnDeclaration, TableDeclaration } from '@pepys/kit';
+import { match } from 'ts-pattern';
+
+export function buildEntityRow(
+  table: TableDeclaration,
+  stored: Readonly<Record<string, unknown>>,
+): Record<string, unknown> {
+  return Object.fromEntries(
+    Object.entries(table.columns).map(([name, column]) => [name, toValue(column, stored[name])]),
+  );
+}
+
+function toValue(column: ColumnDeclaration, stored: unknown): unknown {
+  return match(column.kind)
+    .with('boolean', () => (stored === null ? null : stored === 1))
+    .with('json', () => (typeof stored === 'string' ? parseJSON(stored) : stored))
+    .with('text', 'integer', 'real', 'entityID', 'timestamp', () => stored)
+    .exhaustive();
+}
+
+function parseJSON(text: string): unknown {
+  return JSON.parse(text);
+}

package/src/queries/get-entity/find-entity-consumer.ts

@@ -0,0 +1,23 @@
+import type { ConsumerDeclaration, EntityIDParts, EntityViewDeclaration } from '@pepys/kit';
+
+export interface EntityConsumer {
+  consumer: ConsumerDeclaration;
+  entityView: EntityViewDeclaration;
+}
+
+export function findEntityConsumer(
+  consumers: readonly ConsumerDeclaration[],
+  parts: EntityIDParts,
+): EntityConsumer | undefined {
+  const kind = `${parts.source}:${parts.kind}`;
+
+  for (const consumer of consumers) {
+    const entityView = consumer.entityViews.find((view) => view.kind === kind);
+
+    if (entityView !== undefined) {
+      return { consumer, entityView };
+    }
+  }
+
+  return undefined;
+}

package/src/queries/get-entity/read-entity.ts

@@ -0,0 +1,70 @@
+import type { CoreOutputs } from '@pepys/contract';
+import type { EntityID, EntityIDParts, Result } from '@pepys/kit';
+import { sql } from 'kysely';
+import invariant from 'tiny-invariant';
+import { z } from 'zod';
+import type { QueryContext, QueryFailure } from '../types.ts';
+import { buildEntityRow } from './build-entity-row.ts';
+import { findEntityConsumer } from './find-entity-consumer.ts';
+
+const storedRowSchema = z.record(z.string(), z.unknown());
+const storedRowsSchema = z.object({ rows: z.array(storedRowSchema) });
+
+interface ParsedEntityID {
+  id: EntityID;
+  parts: EntityIDParts;
+}
+
+export async function readEntity(
+  context: QueryContext,
+  parsed: ParsedEntityID,
+): Promise<Result<CoreOutputs['get_entity'], QueryFailure>> {
+  const id = parsed.id;
+
+  const notFound = {
+    ok: false,
+    failure: { code: 'NOT_FOUND', message: `no entity ${id}` },
+  } as const;
+
+  const found = findEntityConsumer(context.consumers, parsed.parts);
+
+  if (found === undefined) {
+    return notFound;
+  }
+
+  const read = await context.store.withConsumer(found.consumer.name, async (transaction) => {
+    const entity = await transaction
+      .selectFrom('entity')
+      .selectAll()
+      .where('id', '=', id)
+      .executeTakeFirst();
+
+    if (entity === undefined) {
+      return notFound;
+    }
+
+    const table = sql.table(found.entityView.table);
+    const idColumn = sql.ref(found.entityView.id);
+    const query = sql`select * from ${table} where ${idColumn} = ${id}`;
+
+    const result = await query.execute(transaction);
+
+    const [stored] = storedRowsSchema.parse(result).rows;
+    const declaration = found.consumer.tables[found.entityView.table];
+
+    invariant(
+      stored !== undefined && declaration !== undefined,
+      'an entity view row has a table row',
+    );
+
+    const value: CoreOutputs['get_entity'] = {
+      entity,
+      row: buildEntityRow(declaration, stored),
+      aliases: [],
+    };
+
+    return { ok: true, value } as const;
+  });
+
+  return read.ok ? read.value : notFound;
+}

package/src/queries/types.ts

@@ -0,0 +1,12 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import type { StoreReader } from '../store-reader/store-reader.ts';
+
+export interface QueryContext {
+  store: StoreReader;
+  consumers: readonly ConsumerDeclaration[];
+}
+
+export interface QueryFailure {
+  code: 'NOT_FOUND';
+  message: string;
+}

package/src/store-reader/create-store-reader.ts

@@ -0,0 +1,123 @@
+import { realpath } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Result } from '@pepys/kit';
+import { createTursoDialect } from '@pepys/kysely-turso';
+import type { Database } from '@tursodatabase/database';
+import { Kysely } from 'kysely';
+import type { Transaction } from 'kysely';
+import type { ConsumerDB, StoreReader } from './store-reader.ts';
+
+interface OpenConsumer {
+  target: string;
+  database: Kysely<ConsumerDB>;
+  readers: number;
+  isRetired: boolean;
+}
+
+interface StoreReaderOptions {
+  storeDirectory: string;
+  openDatabase: (path: string) => Promise<Database>;
+}
+
+export function createStoreReader(options: StoreReaderOptions): StoreReader {
+  const open = new Map<string, OpenConsumer>();
+  const retired = new Set<OpenConsumer>();
+
+  const withConsumer = async <Value>(
+    consumerName: string,
+    work: (transaction: Transaction<ConsumerDB>) => Promise<Value>,
+  ): Promise<Result<Value, { reason: 'no-current-version' }>> => {
+    // the link swings to a new version directory when `pepys up` swaps, and the old file stays
+    const target = await readCurrentTarget(join(options.storeDirectory, consumerName, 'current'));
+
+    if (target === null) {
+      return { ok: false, failure: { reason: 'no-current-version' } };
+    }
+
+    const opened = openConsumer(consumerName, target);
+    const consumer = opened.consumer;
+
+    consumer.readers += 1;
+
+    if (opened.replaced !== undefined) {
+      await destroyIfDrained(opened.replaced);
+    }
+
+    try {
+      // one transaction keeps every read of a call on one connection and one snapshot
+      return { ok: true, value: await consumer.database.transaction().execute(work) };
+    } finally {
+      consumer.readers -= 1;
+
+      await destroyIfDrained(consumer);
+    }
+  };
+
+  const openConsumer = (
+    consumerName: string,
+    target: string,
+  ): { consumer: OpenConsumer; replaced?: OpenConsumer } => {
+    const existing = open.get(consumerName);
+
+    if (existing?.target === target) {
+      return { consumer: existing };
+    }
+
+    const consumer: OpenConsumer = {
+      target,
+      database: new Kysely<ConsumerDB>({
+        dialect: createTursoDialect({
+          database: () => options.openDatabase(join(target, 'consumer.db')),
+        }),
+      }),
+      readers: 0,
+      isRetired: false,
+    };
+
+    open.set(consumerName, consumer);
+
+    if (existing === undefined) {
+      return { consumer };
+    }
+
+    existing.isRetired = true;
+
+    retired.add(existing);
+
+    return { consumer, replaced: existing };
+  };
+
+  const destroyIfDrained = async (consumer: OpenConsumer): Promise<void> => {
+    if (consumer.isRetired && consumer.readers === 0 && retired.delete(consumer)) {
+      await consumer.database.destroy();
+    }
+  };
+
+  const close = async (): Promise<void> => {
+    const consumers = [...open.values(), ...retired];
+
+    open.clear();
+    retired.clear();
+
+    // a destroy waits for the reads still running on the database
+    await Promise.all(consumers.map((consumer) => consumer.database.destroy()));
+  };
+
+  return { withConsumer, close };
+}
+
+async function readCurrentTarget(currentLink: string): Promise<string | null> {
+  try {
+    return await realpath(currentLink);
+  } catch (error: unknown) {
+    if (isMissingFileError(error)) {
+      return null;
+    }
+
+    throw error;
+  }
+}
+
+function isMissingFileError(error: unknown): boolean {
+  return error instanceof Error && 'code' in error && error.code === 'ENOENT';
+}

package/src/store-reader/open-consumer-database.ts

@@ -0,0 +1,6 @@
+import { connect } from '@tursodatabase/database';
+import type { Database } from '@tursodatabase/database';
+
+export function openConsumerDatabase(path: string): Promise<Database> {
+  return connect(path, { readonly: true, experimental: ['multiprocess_wal'] });
+}

package/src/store-reader/read-change-operations.ts

@@ -0,0 +1,35 @@
+import type { Result } from '@pepys/kit';
+import { z } from 'zod';
+import type { StoreReader } from './store-reader.ts';
+
+interface ChangeOperation {
+  key: string;
+  operation: string;
+}
+
+// each change log row holds one batch as a JSON list of State Protocol messages
+const headersSchema = z.object({ operation: z.string() });
+const changesSchema = z.array(z.object({ key: z.string(), headers: headersSchema }));
+
+export async function readChangeOperations(
+  store: StoreReader,
+  consumerName: string,
+): Promise<Result<ChangeOperation[], { reason: 'no-current-version' }>> {
+  const result = await store.withConsumer(consumerName, (transaction) =>
+    transaction.selectFrom('change_log').select('changes').orderBy('batch').execute(),
+  );
+
+  if (!result.ok) {
+    return result;
+  }
+
+  const operations = result.value.flatMap((row) => {
+    const changes: unknown = JSON.parse(row.changes);
+
+    return changesSchema
+      .parse(changes)
+      .map((change) => ({ key: change.key, operation: change.headers.operation }));
+  });
+
+  return { ok: true, value: operations };
+}

package/src/store-reader/read-offsets.ts

@@ -0,0 +1,28 @@
+import type { Result } from '@pepys/kit';
+import type { StoreReader } from './store-reader.ts';
+
+interface Offset {
+  stream: string;
+  lastSeqNum: number;
+}
+
+export async function readOffsets(
+  store: StoreReader,
+  consumerName: string,
+): Promise<Result<Offset[], { reason: 'no-current-version' }>> {
+  const result = await store.withConsumer(consumerName, (transaction) =>
+    transaction
+      .selectFrom('offsets')
+      .select(['stream', 'last_seq_num'])
+      .orderBy('stream')
+      .execute(),
+  );
+
+  if (!result.ok) {
+    return result;
+  }
+
+  const offsets = result.value.map((row) => ({ stream: row.stream, lastSeqNum: row.last_seq_num }));
+
+  return { ok: true, value: offsets };
+}

package/src/store-reader/read-pull-requests.ts

@@ -0,0 +1,13 @@
+import type { Result } from '@pepys/kit';
+import type { Selectable } from 'kysely';
+import type { ConsumerDB, StoreReader } from './store-reader.ts';
+
+type PullRequestRow = Selectable<ConsumerDB['github_pull_request']>;
+
+export function readPullRequests(
+  store: StoreReader,
+): Promise<Result<PullRequestRow[], { reason: 'no-current-version' }>> {
+  return store.withConsumer('github_pull_request', (transaction) =>
+    transaction.selectFrom('github_pull_request').selectAll().orderBy('id').execute(),
+  );
+}

package/src/store-reader/store-reader.ts

@@ -0,0 +1,37 @@
+import type { Result } from '@pepys/kit';
+import type { DB } from '@pepys/schema';
+import type { Transaction } from 'kysely';
+
+interface EntityViewRow {
+  id: string;
+  kind: string;
+  title: string;
+  url: string;
+  actor: string | null;
+  updated_at: string;
+}
+
+// the host creates these two tables in every consumer file, beside the consumer's own
+interface OffsetRow {
+  stream: string;
+  last_seq_num: number;
+}
+
+interface ChangeLogRow {
+  batch: number;
+  changes: string;
+}
+
+export type ConsumerDB = DB & {
+  entity: EntityViewRow;
+  offsets: OffsetRow;
+  change_log: ChangeLogRow;
+};
+
+export interface StoreReader {
+  withConsumer: <Value>(
+    consumerName: string,
+    work: (transaction: Transaction<ConsumerDB>) => Promise<Value>,
+  ) => Promise<Result<Value, { reason: 'no-current-version' }>>;
+  close: () => Promise<void>;
+}