@pepys/kit 0.0.0-stage → 0.0.1

package/src/links/link-check-cases.json

@@ -0,0 +1,274 @@
+{
+  "consumer": {
+    "name": "archive_shelf",
+    "links": [
+      {
+        "name": "holds",
+        "from": ["archive:shelf"],
+        "to": ["archive:box", "archive:crate"],
+        "hub": false
+      },
+      {
+        "name": "stands_in",
+        "from": ["archive:shelf"],
+        "to": ["archive:room"],
+        "hub": true
+      }
+    ]
+  },
+  "cases": [
+    {
+      "title": "accepts a declared link with both ends of an allowed kind",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": null
+    },
+    {
+      "title": "accepts an end of the second allowed kind",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:crate:c-9",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": null
+    },
+    {
+      "title": "accepts a link to an entity whose row has not arrived",
+      "link": {
+        "type": "stands_in",
+        "from": "archive:shelf:s-1",
+        "to": "archive:room:never-filed",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": null
+    },
+    {
+      "title": "accepts a confidence of 0",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 0,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": null
+    },
+    {
+      "title": "accepts a confidence of 1",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": null
+    },
+    {
+      "title": "accepts an evidence that is not a fact ID",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "x"
+      },
+      "problem": null
+    },
+    {
+      "title": "refuses a type the consumer does not declare",
+      "link": {
+        "type": "leans_on",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-declared"
+    },
+    {
+      "title": "refuses a from end that is not an entity ID",
+      "link": {
+        "type": "holds",
+        "from": "archive",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "refuses a to end with no ID part",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "refuses a to end with an empty ID",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "refuses a from end whose source is not lower snake case",
+      "link": {
+        "type": "holds",
+        "from": "Archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "refuses a fact ID as an end",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "event:archive:e-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "refuses a from end of a kind the type does not allow",
+      "link": {
+        "type": "holds",
+        "from": "archive:box:b-2",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "wrong-kind"
+    },
+    {
+      "title": "refuses a to end of a kind the type does not allow",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:room:r-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "wrong-kind"
+    },
+    {
+      "title": "refuses a kind that only starts with an allowed kind",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box_lid:l-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "wrong-kind"
+    },
+    {
+      "title": "refuses an allowed kind of another source",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "depot:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "wrong-kind"
+    },
+    {
+      "title": "refuses a confidence below 0",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": -0.1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "bad-confidence"
+    },
+    {
+      "title": "refuses a confidence above 1",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1.5,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "bad-confidence"
+    },
+    {
+      "title": "refuses an empty evidence",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": ""
+      },
+      "problem": "no-evidence"
+    },
+    {
+      "title": "reports the type before a bad end",
+      "link": {
+        "type": "leans_on",
+        "from": "archive",
+        "to": "archive:box:b-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-declared"
+    },
+    {
+      "title": "reports the from end before the to end",
+      "link": {
+        "type": "holds",
+        "from": "archive",
+        "to": "archive:room:r-1",
+        "confidence": 1,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "not-an-entity-id"
+    },
+    {
+      "title": "reports a wrong kind before a bad confidence",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:room:r-1",
+        "confidence": 2,
+        "evidence": "event:archive:e-1"
+      },
+      "problem": "wrong-kind"
+    },
+    {
+      "title": "reports a bad confidence before an empty evidence",
+      "link": {
+        "type": "holds",
+        "from": "archive:shelf:s-1",
+        "to": "archive:box:b-1",
+        "confidence": 2,
+        "evidence": ""
+      },
+      "problem": "bad-confidence"
+    }
+  ]
+}

package/src/links/types.ts

@@ -0,0 +1,22 @@
+// What is wrong with one link that a consumer returns. The host has no deployment registry, so a
+// type is declared by the consumer that writes it, or by nothing.
+export type LinkProblem =
+  | { kind: 'not-declared'; type: string; consumer: string }
+  | { kind: 'not-an-entity-id'; end: 'from' | 'to'; id: string }
+  | { kind: 'wrong-kind'; end: 'from' | 'to'; id: string; allowed: readonly string[] }
+  | { kind: 'bad-confidence'; confidence: number }
+  | { kind: 'no-evidence' };
+
+// A consumer as its declaration or its manifest holds it, so the shared case file fits it too.
+export interface LinkCheckConsumer {
+  name: string;
+  links: readonly { name: string; from: readonly string[]; to: readonly string[] }[];
+}
+
+export interface CheckedLink {
+  type: string;
+  from: string;
+  to: string;
+  confidence: number;
+  evidence: string;
+}

package/src/queries/define-query.ts

@@ -0,0 +1,14 @@
+import type { z } from 'zod';
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import type { QueryDefinition } from './types.ts';
+
+export function defineQuery<
+  const Name extends string,
+  InputSchema extends z.ZodType,
+  OutputSchema extends z.ZodType,
+  const Consumers extends readonly ConsumerDeclaration[],
+>(
+  query: QueryDefinition<Name, InputSchema, OutputSchema, Consumers>,
+): QueryDefinition<Name, InputSchema, OutputSchema, Consumers> {
+  return query;
+}

package/src/queries/types.ts

@@ -0,0 +1,56 @@
+import type { Kysely } from 'kysely';
+import type { Simplify, UnionToIntersection } from 'type-fest';
+import type { z } from 'zod';
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import type { ConsumersDB } from '../store-reader/store-reader.ts';
+import type { TableRow } from '../tables/table-row.ts';
+
+// The Kysely tables of a set of consumers, from their declarations.
+export type StoreTables<Consumers extends readonly ConsumerDeclaration[]> = Simplify<
+  UnionToIntersection<
+    {
+      [Index in keyof Consumers]: {
+        -readonly [Table in keyof Consumers[Index]['tables']]: TableRow<
+          Consumers[Index]['tables'][Table]
+        >;
+      };
+    }[number]
+  >
+>;
+
+export interface QueryCall<Input, Consumers extends readonly ConsumerDeclaration[]> {
+  input: Input;
+
+  // One read-only connection with the current file of each consumer in `reads` attached, and the
+  // `entity` and `link_effective` views over those files.
+  db: Kysely<ConsumersDB<Consumers>>;
+}
+
+// Generic over the schemas, not their types: `z.ZodType<Input>` keeps the parsed type alone, and a
+// client needs the type a schema takes in.
+export interface QueryDefinition<
+  Name extends string = string,
+  InputSchema extends z.ZodType = z.ZodType,
+  OutputSchema extends z.ZodType = z.ZodType,
+  Consumers extends readonly ConsumerDeclaration[] = readonly ConsumerDeclaration[],
+> {
+  // snake case: the procedure key and the MCP tool name
+  readonly name: Name;
+
+  // the text an MCP agent reads
+  readonly description: string;
+
+  // plain zod objects with no transform, because the MCP schemas come from them
+  readonly input: InputSchema;
+  readonly output: OutputSchema;
+  readonly reads: Consumers;
+  readonly handler: (
+    call: QueryCall<z.output<InputSchema>, Consumers>,
+  ) => Promise<z.input<OutputSchema>>;
+}
+
+// Any query, as a list of a plug-in's queries holds it. A handler takes its own call type, so no
+// one call type fits every handler.
+export type AnyQueryDefinition = Omit<QueryDefinition, 'handler'> & {
+  readonly handler: (call: never) => Promise<unknown>;
+};

package/src/send/create-fact-batch-sender.ts

@@ -0,0 +1,143 @@
+import { z } from 'zod';
+import { buildDeliveryProof } from '../delivery-checks/build-delivery-proof.ts';
+import type { FactBatchBody, ReceivePath, SourceDefinition } from '../sources/types.ts';
+import type { Result } from '../types.ts';
+import type { OutgoingFact, SendFailure } from './types.ts';
+
+interface FactBatchSenderOptions<Source extends SourceDefinition> {
+  source: Source;
+  url: string;
+  secret: string;
+
+  // Defaults to the global `fetch`. A route test passes ingest's own handler.
+  fetch?: ((request: Request) => Promise<Response>) | undefined;
+}
+
+type FactBatchSender<Source extends SourceDefinition> = (
+  facts: readonly OutgoingFact<Source>[],
+) => Promise<Result<{ stored: number }, SendFailure>>;
+
+const defaultFactLimit = 100;
+
+// Signs each batch as the `check` of the source's receive path says, and sends it. A `retry` is a
+// 502, 503 or 504 answer, a timeout or no connection: send the same batch again, unchanged. A
+// `refused` is any other answer that is no success, 500 included: never resend that batch as it is.
+export function createFactBatchSender<Source extends SourceDefinition>(
+  options: FactBatchSenderOptions<Source>,
+): FactBatchSender<Source> {
+  const path = options.source.live;
+
+  if (path?.kind !== 'receive') {
+    throw new Error(`the source ${options.source.name} has no receive path to send to`);
+  }
+
+  if (options.secret === '') {
+    throw new Error(`the secret for the source ${options.source.name} is empty`);
+  }
+
+  const send = options.fetch ?? fetch;
+
+  return async (facts) => {
+    const factLimit = path.factLimit ?? defaultFactLimit;
+
+    if (facts.length > factLimit) {
+      return {
+        ok: false,
+        failure: {
+          kind: 'refused',
+          status: 413,
+          reason: `${facts.length} facts is over the path's limit of ${factLimit}`,
+        },
+      };
+    }
+
+    const request = buildRequest(path, options.url, options.secret, facts);
+
+    const outcome = await sendRequest(send, request);
+
+    return outcome;
+  };
+}
+
+const sendTimeoutMs = 30_000;
+
+function buildRequest(
+  path: ReceivePath,
+  url: string,
+  secret: string,
+  facts: readonly FactBatchBody['facts'][number][],
+): Request {
+  const batch: FactBatchBody = { v: 1, facts };
+
+  // the bytes that are signed are the bytes that are sent
+  const body = new TextEncoder().encode(JSON.stringify(batch));
+
+  return new Request(url, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      [path.check.header]: buildDeliveryProof(path.check, secret, body),
+    },
+    body,
+    redirect: 'manual',
+    signal: AbortSignal.timeout(sendTimeoutMs),
+  });
+}
+
+async function sendRequest(
+  send: (request: Request) => Promise<Response>,
+  request: Request,
+): Promise<Result<{ stored: number }, SendFailure>> {
+  let answer: { status: number; text: string };
+
+  try {
+    const response = await send(request);
+    const text = await response.text();
+
+    answer = { status: response.status, text };
+  } catch (error) {
+    return { ok: false, failure: { kind: 'retry', reason: String(error) } };
+  }
+
+  return pickSendResult(answer.status, answer.text);
+}
+
+const answerSchema = z.union([
+  z.object({ status: z.literal('stored'), stored: z.number().int().nonnegative() }),
+  z.object({ status: z.literal('ignored') }),
+]);
+
+// Ingest answers 500 for a batch that no resend of the same bytes can store, so only these mean
+// that the same batch may succeed later.
+const retryStatuses: ReadonlySet<number> = new Set([502, 503, 504]);
+
+function pickSendResult(status: number, text: string): Result<{ stored: number }, SendFailure> {
+  if (retryStatuses.has(status)) {
+    return { ok: false, failure: { kind: 'retry', reason: `ingest answered ${status}: ${text}` } };
+  }
+
+  if (status < 200 || status >= 300) {
+    return { ok: false, failure: { kind: 'refused', status, reason: text } };
+  }
+
+  const json = parseJSON(text);
+  const answer = answerSchema.safeParse(json);
+
+  // A success that names no count may still have stored the batch, and a repeat is safe, because
+  // every consumer drops a copy by its fact ID.
+  if (!answer.success) {
+    return { ok: false, failure: { kind: 'retry', reason: `ingest answered ${status}: ${text}` } };
+  }
+
+  return { ok: true, value: { stored: answer.data.status === 'stored' ? answer.data.stored : 0 } };
+}
+
+function parseJSON(text: string): unknown {
+  try {
+    const value: unknown = JSON.parse(text);
+
+    return value;
+  } catch {
+    return undefined;
+  }
+}

package/src/send/types.ts

@@ -0,0 +1,25 @@
+import type { z } from 'zod';
+import type { FactBatchBody, SourceDefinition } from '../sources/types.ts';
+
+// One fact of a batch, typed by the families of the source: the type picks the payload schema, and
+// the entity has the family's kind. It is a fact of the wire body whatever the source.
+export type OutgoingFact<Source extends SourceDefinition> = FactBatchBody['facts'][number] &
+  {
+    [Family in Source['families'][number] as Family['stream']]: {
+      [Type in keyof Family['types'] & string]: {
+        type: Type;
+        event: string;
+        entity: `${Family['entityKind']}:${string}`;
+        actor?: string | undefined;
+        at: string;
+        payload: z.input<Family['types'][Type]>;
+      };
+    }[keyof Family['types'] & string];
+  }[Source['families'][number]['stream']];
+
+// Every 5xx is a `retry`: ingest answers 500 when the log refuses an append for a fault of the
+// deployment, and part of the batch may be stored by then. A sender that dropped the batch as
+// refused would lose the rest.
+export type SendFailure =
+  | { kind: 'refused'; status: number; reason: string }
+  | { kind: 'retry'; reason: string };

package/src/send.ts

@@ -0,0 +1,2 @@
+export { createFactBatchSender } from './send/create-fact-batch-sender.ts';
+export type * from './send/types.ts';

package/src/sources/define-fact-family.ts

@@ -0,0 +1,10 @@
+import type { z } from 'zod';
+import type { EntityKind } from '../link-types/types.ts';
+import type { FactFamily } from './types.ts';
+
+export function defineFactFamily<
+  const Types extends Readonly<Record<string, z.ZodType>>,
+  const Kind extends EntityKind,
+>(family: FactFamily<Types, Kind>): FactFamily<Types, Kind> {
+  return family;
+}

package/src/sources/define-source.ts

@@ -0,0 +1,7 @@
+import type { FactFamily, SourceDefinition } from './types.ts';
+
+export function defineSource<const Family extends FactFamily>(
+  source: SourceDefinition<Family>,
+): SourceDefinition<Family> {
+  return source;
+}

package/src/sources/read-fact-batch.ts

@@ -0,0 +1,114 @@
+import { toTagged } from '@zgeoff/std';
+import { z } from 'zod';
+import { buildEventFactID } from '../facts/build-event-fact-id.ts';
+import type { EntityID, StreamName } from '../ids/types.ts';
+import type { Result } from '../types.ts';
+import type {
+  Delivery,
+  FactBatchBody,
+  FactFamily,
+  PendingFact,
+  Received,
+  SourceContext,
+} from './types.ts';
+
+const batchFactSchema = z.strictObject({
+  type: z.string().min(1),
+  event: z.string().min(1),
+  entity: z.string(),
+  actor: z.string().optional(),
+  at: z.string(),
+  payload: z.unknown(),
+});
+
+const factBatchBodySchema = z.strictObject({
+  v: z.literal(1),
+  facts: z.array(batchFactSchema),
+}) satisfies z.ZodType<FactBatchBody>;
+
+// The read of a fact batch path. The testing package builds its envelopes with it, as ingest does.
+export function readFactBatch(delivery: Delivery, source: SourceContext): Received {
+  const json = parseJSON(new TextDecoder().decode(delivery.body));
+
+  if (!json.ok) {
+    return { kind: 'refused', status: 400, reason: 'the body is not JSON' };
+  }
+
+  const body = factBatchBodySchema.safeParse(json.value);
+
+  if (!body.success) {
+    return { kind: 'refused', status: 422, reason: z.prettifyError(body.error) };
+  }
+
+  if (body.data.facts.length === 0) {
+    return { kind: 'ignored' };
+  }
+
+  const facts: PendingFact[] = [];
+
+  for (const [index, fact] of body.data.facts.entries()) {
+    const built = buildPendingFact(fact, source);
+
+    if (!built.ok) {
+      return { kind: 'refused', status: 422, reason: `fact ${index}: ${built.failure}` };
+    }
+
+    facts.push(built.value);
+  }
+
+  return { kind: 'batch', batch: { facts } };
+}
+
+function parseJSON(text: string): Result<unknown, null> {
+  try {
+    const value: unknown = JSON.parse(text);
+
+    return { ok: true, value };
+  } catch {
+    return { ok: false, failure: null };
+  }
+}
+
+type BatchFact = FactBatchBody['facts'][number];
+
+function buildPendingFact(fact: BatchFact, source: SourceContext): Result<PendingFact, string> {
+  const family = findFamily(source.families, fact.type);
+
+  if (family === undefined) {
+    return { ok: false, failure: `the source has no fact type ${fact.type}` };
+  }
+
+  if (!fact.entity.startsWith(`${family.entityKind}:`)) {
+    return { ok: false, failure: `the entity is not of the kind ${family.entityKind}` };
+  }
+
+  const payload = family.types[fact.type]?.safeParse(fact.payload);
+
+  if (payload?.success !== true) {
+    return { ok: false, failure: `the payload does not fit the type ${fact.type}` };
+  }
+
+  return {
+    ok: true,
+    value: {
+      stream: toTagged<StreamName>(family.stream),
+      fact: {
+        id: buildEventFactID({ source: source.name, event: fact.event }),
+        type: fact.type,
+        v: 1,
+        api: family.api,
+        entity: toTagged<EntityID>(fact.entity),
+        ...(fact.actor === undefined ? {} : { actor: toTagged<EntityID>(fact.actor) }),
+        at: fact.at,
+
+        // the payload as sent: a schema may strip or reshape, and the log keeps what arrived
+        payload: fact.payload,
+      },
+      blobs: [],
+    },
+  };
+}
+
+function findFamily(families: readonly FactFamily[], type: string): FactFamily | undefined {
+  return families.find((family) => Object.hasOwn(family.types, type));
+}

package/src/sources/receive-fact-batch.ts

@@ -0,0 +1,11 @@
+import { readFactBatch } from './read-fact-batch.ts';
+import type { ReceivePath } from './types.ts';
+
+type FactBatchReceiver = Pick<
+  ReceivePath,
+  'route' | 'check' | 'bodyByteLimit' | 'factLimit' | 'reach' | 'sender'
+>;
+
+export function receiveFactBatch(receiver: FactBatchReceiver): ReceivePath {
+  return { ...receiver, kind: 'receive', read: readFactBatch };
+}