@pepys/kit 0.0.1 → 0.1.1

package/README.md

@@ -34,7 +34,8 @@ Test the plug-in with [`@pepys/testing`](../testing/README.md), and build it wit
 
 ## Features
 
-- `@pepys/kit`: the `define*` declarations and the deployment check.
+- `@pepys/kit`: the `define*` declarations and the deployment check. `toStreamName` and
+  `toStreamOffset` give a family's `stream` and a number as the branded types that a fact log takes.
 - `@pepys/kit/generators`: the SQL schema and glue generators.
 - `@pepys/kit/send`: the client that signs and sends a fact batch.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/kit",
-  "version": "0.0.1",
+  "version": "0.1.1",
   "description": "Declare a pepys plug-in or deployment: fact families, sources, consumers, tables, links and queries",
   "license": "MIT",
   "repository": {

package/src/deployments/check-deployment.ts

@@ -271,6 +271,7 @@ const hostTableNames: ReadonlySet<string> = new Set([
   'change_log',
   'applied_facts',
   'scope_row',
+  'scope_fact',
 ]);
 
 function collectReservedNames(

package/src/deployments/types.ts

@@ -40,9 +40,11 @@ export interface Deployment<Plugins extends readonly Plugin[] = readonly Plugin[
   readonly settings?: Readonly<Record<string, Readonly<Record<string, SettingValue>>>> | undefined;
 }
 
-// The typed client of one plug-in's queries. A type only, so the kit needs no oRPC.
+// The typed client of one plug-in's queries. A type only, so the kit needs no oRPC. A plug-in with
+// no queries, written inside `defineDeployment`, takes its query type from the call's context, so
+// a query name that is no literal gives no method rather than an index signature.
 export type PluginClient<Target extends Plugin> = {
-  [Query in Target['queries'][number] as Query['name']]: (
+  [Query in Target['queries'][number] as string extends Query['name'] ? never : Query['name']]: (
     input: z.input<Query['input']>,
   ) => Promise<z.output<Query['output']>>;
 };

package/src/generators/build-payload-shape.ts

@@ -60,6 +60,11 @@ function buildPayloadField(name: string, schema: z.core.$ZodType): PayloadField
       shape: buildPayloadShape(optional.unwrap()),
       isOptional: true,
     }))
+    .with(P.instanceOf(z.ZodExactOptional), (optional) => ({
+      name,
+      shape: buildPayloadShape(optional.unwrap()),
+      isOptional: true,
+    }))
     .with(P.instanceOf(z.ZodDefault), (withDefault) => ({
       name,
       shape: buildPayloadShape(withDefault.unwrap()),

package/src/ids/to-stream-name.ts

@@ -0,0 +1,7 @@
+import { toTagged } from '@zgeoff/std';
+import type { DeclaredStreamName } from '../types.ts';
+import type { StreamName } from './types.ts';
+
+export function toStreamName(stream: DeclaredStreamName): StreamName {
+  return toTagged<StreamName>(stream);
+}

package/src/ids/to-stream-offset.ts

@@ -0,0 +1,6 @@
+import { toTagged } from '@zgeoff/std';
+import type { StreamOffset } from './types.ts';
+
+export function toStreamOffset(offset: number): StreamOffset {
+  return toTagged<StreamOffset>(offset);
+}

package/src/index.ts

@@ -1,4 +1,5 @@
 export { buildEnrichmentFactID } from './facts/build-enrichment-fact-id.ts';
+export { buildLinkRegistry } from './links/build-link-registry.ts';
 export { buildEntityID } from './entity-ids/build-entity-id.ts';
 export { buildEventFactID } from './facts/build-event-fact-id.ts';
 export { buildManualFactID } from './facts/build-manual-fact-id.ts';
@@ -24,6 +25,8 @@ export { parseStreamName } from './facts/parse-stream-name.ts';
 export { readFactBatch } from './sources/read-fact-batch.ts';
 export { receiveFactBatch } from './sources/receive-fact-batch.ts';
 export type { TableRow } from './tables/table-row.ts';
+export { toStreamName } from './ids/to-stream-name.ts';
+export { toStreamOffset } from './ids/to-stream-offset.ts';
 export type * from './consumers/types.ts';
 export type * from './deployments/types.ts';
 export type * from './ids/types.ts';

package/src/links/build-link-registry.ts

@@ -0,0 +1,7 @@
+import type { LinkTypeDeclaration } from '../link-types/types.ts';
+import type { LinkRegistry } from './types.ts';
+
+// The deployment check refuses one name with two shapes, so a name maps to one declaration.
+export function buildLinkRegistry(linkTypes: readonly LinkTypeDeclaration[]): LinkRegistry {
+  return new Map(linkTypes.map((linkType) => [linkType.name, linkType]));
+}

package/src/links/types.ts

@@ -1,3 +1,8 @@
+import type { LinkTypeDeclaration } from '../link-types/types.ts';
+
+// Each link type of a deployment, by name.
+export type LinkRegistry = ReadonlyMap<string, LinkTypeDeclaration>;
+
 // What is wrong with one link that a consumer returns. The host has no deployment registry, so a
 // type is declared by the consumer that writes it, or by nothing.
 export type LinkProblem =

package/src/queries/types.ts

@@ -5,18 +5,23 @@ import type { ConsumerDeclaration } from '../consumers/types.ts';
 import type { ConsumersDB } from '../store-reader/store-reader.ts';
 import type { TableRow } from '../tables/table-row.ts';
 
-// The Kysely tables of a set of consumers, from their declarations.
-export type StoreTables<Consumers extends readonly ConsumerDeclaration[]> = Simplify<
-  UnionToIntersection<
-    {
-      [Index in keyof Consumers]: {
-        -readonly [Table in keyof Consumers[Index]['tables']]: TableRow<
-          Consumers[Index]['tables'][Table]
-        >;
-      };
-    }[number]
-  >
->;
+// The Kysely tables of a set of consumers, from their declarations. No consumers give `unknown`,
+// which adds no table to an intersection, where the intersection of an empty union would give
+// `never`.
+export type StoreTables<Consumers extends readonly ConsumerDeclaration[]> =
+  Consumers extends readonly []
+    ? unknown
+    : Simplify<
+        UnionToIntersection<
+          {
+            [Index in keyof Consumers]: {
+              -readonly [Table in keyof Consumers[Index]['tables']]: TableRow<
+                Consumers[Index]['tables'][Table]
+              >;
+            };
+          }[number]
+        >
+      >;
 
 export interface QueryCall<Input, Consumers extends readonly ConsumerDeclaration[]> {
   input: Input;

package/src/send/types.ts

@@ -17,9 +17,9 @@ export type OutgoingFact<Source extends SourceDefinition> = FactBatchBody['facts
     }[keyof Family['types'] & string];
   }[Source['families'][number]['stream']];
 
-// Every 5xx is a `retry`: ingest answers 500 when the log refuses an append for a fault of the
-// deployment, and part of the batch may be stored by then. A sender that dropped the batch as
-// refused would lose the rest.
+// A 502, 503 or 504 answer, a timeout and no connection are a `retry`: send the same batch again,
+// unchanged. Every other answer that is no success is `refused`, 500 included: ingest answers 500
+// for a batch that no resend of the same bytes can store.
 export type SendFailure =
   | { kind: 'refused'; status: number; reason: string }
   | { kind: 'retry'; reason: string };

package/src/sources/read-fact-batch.ts

@@ -1,4 +1,4 @@
-import { toTagged } from '@zgeoff/std';
+import { isRecord, toTagged } from '@zgeoff/std';
 import { z } from 'zod';
 import { buildEventFactID } from '../facts/build-event-fact-id.ts';
 import type { EntityID, StreamName } from '../ids/types.ts';
@@ -9,6 +9,7 @@ import type {
   FactFamily,
   PendingFact,
   Received,
+  Sender,
   SourceContext,
 } from './types.ts';
 
@@ -47,7 +48,7 @@ export function readFactBatch(delivery: Delivery, source: SourceContext): Receiv
   const facts: PendingFact[] = [];
 
   for (const [index, fact] of body.data.facts.entries()) {
-    const built = buildPendingFact(fact, source);
+    const built = buildPendingFact(fact, source, delivery.sender);
 
     if (!built.ok) {
       return { kind: 'refused', status: 422, reason: `fact ${index}: ${built.failure}` };
@@ -71,7 +72,11 @@ function parseJSON(text: string): Result<unknown, null> {
 
 type BatchFact = FactBatchBody['facts'][number];
 
-function buildPendingFact(fact: BatchFact, source: SourceContext): Result<PendingFact, string> {
+function buildPendingFact(
+  fact: BatchFact,
+  source: SourceContext,
+  sender: Sender | undefined,
+): Result<PendingFact, string> {
   const family = findFamily(source.families, fact.type);
 
   if (family === undefined) {
@@ -82,7 +87,13 @@ function buildPendingFact(fact: BatchFact, source: SourceContext): Result<Pendin
     return { ok: false, failure: `the entity is not of the kind ${family.entityKind}` };
   }
 
-  const payload = family.types[fact.type]?.safeParse(fact.payload);
+  const sent = buildSentPayload(fact.payload, sender);
+
+  if (!sent.ok) {
+    return sent;
+  }
+
+  const payload = family.types[fact.type]?.safeParse(sent.value);
 
   if (payload?.success !== true) {
     return { ok: false, failure: `the payload does not fit the type ${fact.type}` };
@@ -102,7 +113,7 @@ function buildPendingFact(fact: BatchFact, source: SourceContext): Result<Pendin
         at: fact.at,
 
         // the payload as sent: a schema may strip or reshape, and the log keeps what arrived
-        payload: fact.payload,
+        payload: sent.value,
       },
       blobs: [],
     },
@@ -112,3 +123,23 @@ function buildPendingFact(fact: BatchFact, source: SourceContext): Result<Pendin
 function findFamily(families: readonly FactFamily[], type: string): FactFamily | undefined {
   return families.find((family) => Object.hasOwn(family.types, type));
 }
+
+// On a path with a proven sender, the stored `origin` is that sender, whatever the body holds: the
+// route's secret proves the batch, and only the proof says which sender wrote it.
+function buildSentPayload(payload: unknown, sender: Sender | undefined): Result<unknown, string> {
+  if (sender === undefined) {
+    return { ok: true, value: payload };
+  }
+
+  if (!isRecord(payload)) {
+    return {
+      ok: false,
+      failure: 'the payload is not an object, so it cannot hold the proven sender',
+    };
+  }
+
+  return {
+    ok: true,
+    value: { ...payload, origin: { issuer: sender.issuer, subject: sender.subject } },
+  };
+}

package/src/sources/types.ts

@@ -1,5 +1,5 @@
 import type { z } from 'zod';
-import type { BlobHash, StreamName } from '../ids/types.ts';
+import type { BlobHash, EntityID, StreamName } from '../ids/types.ts';
 import type { EntityKind } from '../link-types/types.ts';
 import type { DeclaredStreamName, FactEnvelope } from '../types.ts';
 
@@ -122,6 +122,18 @@ export interface PullPath {
 
 export interface PullContext {
   env: Readonly<Record<string, string>>;
+
+  // For a `resync` pull, the source's marked entities when the pull starts; empty for any other
+  // pull. Ingest removes each of these marks once the batch the pull returns is stored, so a pull
+  // that cannot read every marked entity answers `unavailable` and every mark stays.
+  marks: readonly ResyncMark[];
+}
+
+// An entity whose row may be behind the source, with the locator of the last fact ingest stored
+// for it, if that fact had one.
+export interface ResyncMark {
+  entity: EntityID;
+  locator: string | undefined;
 }
 
 export type Pulled =
@@ -151,4 +163,8 @@ export interface PendingFact {
   stream: StreamName;
   fact: FactEnvelope<unknown>;
   blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
+
+  // What a source's resync pull needs to read the entity again, such as an API path. Ingest keeps
+  // it beside the entity's mark, and stores it in no fact.
+  locator?: string | undefined;
 }