@pepys/kit 0.0.0-stage → 0.1.0

package/src/sources/read-fact-batch.ts

@@ -0,0 +1,145 @@
+import { isRecord, toTagged } from '@zgeoff/std';
+import { z } from 'zod';
+import { buildEventFactID } from '../facts/build-event-fact-id.ts';
+import type { EntityID, StreamName } from '../ids/types.ts';
+import type { Result } from '../types.ts';
+import type {
+  Delivery,
+  FactBatchBody,
+  FactFamily,
+  PendingFact,
+  Received,
+  Sender,
+  SourceContext,
+} from './types.ts';
+
+const batchFactSchema = z.strictObject({
+  type: z.string().min(1),
+  event: z.string().min(1),
+  entity: z.string(),
+  actor: z.string().optional(),
+  at: z.string(),
+  payload: z.unknown(),
+});
+
+const factBatchBodySchema = z.strictObject({
+  v: z.literal(1),
+  facts: z.array(batchFactSchema),
+}) satisfies z.ZodType<FactBatchBody>;
+
+// The read of a fact batch path. The testing package builds its envelopes with it, as ingest does.
+export function readFactBatch(delivery: Delivery, source: SourceContext): Received {
+  const json = parseJSON(new TextDecoder().decode(delivery.body));
+
+  if (!json.ok) {
+    return { kind: 'refused', status: 400, reason: 'the body is not JSON' };
+  }
+
+  const body = factBatchBodySchema.safeParse(json.value);
+
+  if (!body.success) {
+    return { kind: 'refused', status: 422, reason: z.prettifyError(body.error) };
+  }
+
+  if (body.data.facts.length === 0) {
+    return { kind: 'ignored' };
+  }
+
+  const facts: PendingFact[] = [];
+
+  for (const [index, fact] of body.data.facts.entries()) {
+    const built = buildPendingFact(fact, source, delivery.sender);
+
+    if (!built.ok) {
+      return { kind: 'refused', status: 422, reason: `fact ${index}: ${built.failure}` };
+    }
+
+    facts.push(built.value);
+  }
+
+  return { kind: 'batch', batch: { facts } };
+}
+
+function parseJSON(text: string): Result<unknown, null> {
+  try {
+    const value: unknown = JSON.parse(text);
+
+    return { ok: true, value };
+  } catch {
+    return { ok: false, failure: null };
+  }
+}
+
+type BatchFact = FactBatchBody['facts'][number];
+
+function buildPendingFact(
+  fact: BatchFact,
+  source: SourceContext,
+  sender: Sender | undefined,
+): Result<PendingFact, string> {
+  const family = findFamily(source.families, fact.type);
+
+  if (family === undefined) {
+    return { ok: false, failure: `the source has no fact type ${fact.type}` };
+  }
+
+  if (!fact.entity.startsWith(`${family.entityKind}:`)) {
+    return { ok: false, failure: `the entity is not of the kind ${family.entityKind}` };
+  }
+
+  const sent = buildSentPayload(fact.payload, sender);
+
+  if (!sent.ok) {
+    return sent;
+  }
+
+  const payload = family.types[fact.type]?.safeParse(sent.value);
+
+  if (payload?.success !== true) {
+    return { ok: false, failure: `the payload does not fit the type ${fact.type}` };
+  }
+
+  return {
+    ok: true,
+    value: {
+      stream: toTagged<StreamName>(family.stream),
+      fact: {
+        id: buildEventFactID({ source: source.name, event: fact.event }),
+        type: fact.type,
+        v: 1,
+        api: family.api,
+        entity: toTagged<EntityID>(fact.entity),
+        ...(fact.actor === undefined ? {} : { actor: toTagged<EntityID>(fact.actor) }),
+        at: fact.at,
+
+        // the payload as sent: a schema may strip or reshape, and the log keeps what arrived
+        payload: sent.value,
+      },
+      blobs: [],
+    },
+  };
+}
+
+function findFamily(families: readonly FactFamily[], type: string): FactFamily | undefined {
+  return families.find((family) => Object.hasOwn(family.types, type));
+}
+
+// On a path with a proven sender, the stored `origin` is that sender, whatever the body holds: the
+// route's secret proves the batch, and only the proof says which sender wrote it.
+function buildSentPayload(payload: unknown, sender: Sender | undefined): Result<unknown, string> {
+  if (sender === undefined) {
+    return { ok: true, value: payload };
+  }
+
+  if (!isRecord(payload)) {
+    return {
+      ok: false,
+      failure: 'the payload is not an object, so it cannot hold the proven sender',
+    };
+  }
+
+  return {
+    ok: true,
+    value: { ...payload, origin: { issuer: sender.issuer, subject: sender.subject } },
+  };
+}

package/src/sources/receive-fact-batch.ts

@@ -0,0 +1,11 @@
+import { readFactBatch } from './read-fact-batch.ts';
+import type { ReceivePath } from './types.ts';
+
+type FactBatchReceiver = Pick<
+  ReceivePath,
+  'route' | 'check' | 'bodyByteLimit' | 'factLimit' | 'reach' | 'sender'
+>;
+
+export function receiveFactBatch(receiver: FactBatchReceiver): ReceivePath {
+  return { ...receiver, kind: 'receive', read: readFactBatch };
+}

package/src/sources/types.ts

@@ -0,0 +1,170 @@
+import type { z } from 'zod';
+import type { BlobHash, EntityID, StreamName } from '../ids/types.ts';
+import type { EntityKind } from '../link-types/types.ts';
+import type { DeclaredStreamName, FactEnvelope } from '../types.ts';
+
+export interface FactFamily<
+  Types extends Readonly<Record<string, z.ZodType>> = Readonly<Record<string, z.ZodType>>,
+  Kind extends EntityKind = EntityKind,
+> {
+  readonly stream: DeclaredStreamName;
+  readonly api: string;
+
+  // the kind of the `entity` of every fact of the family
+  readonly entityKind: Kind;
+
+  // one zod schema per fact type; a fact type belongs to one family in the deployment
+  readonly types: Types;
+}
+
+// A source holds no secret and reads no environment.
+export interface SourceDefinition<Family extends FactFamily = FactFamily> {
+  readonly name: string;
+  readonly families: readonly Family[];
+  readonly live?: ReceivePath | PullPath | undefined;
+  readonly recovery?: PullPath | undefined;
+  readonly backfill?: PullPath | undefined;
+  readonly resync?: PullPath | undefined;
+}
+
+// Ingest owns every step of a delivery but `read`: the method, the size, the check, the sender, the
+// decoding, the fact limit, the appends and the answer.
+export interface ReceivePath {
+  readonly kind: 'receive';
+  readonly route: string;
+
+  // More routes that pass the same check and sender, answer as an ignored delivery, and store
+  // nothing. An OTLP exporter sends metrics and traces beside its logs, and retries a refusal.
+  readonly ignoredRoutes?: readonly string[] | undefined;
+  readonly check: DeliveryCheck;
+
+  // Default 1 MiB, for the body as sent and for the body after decoding.
+  readonly bodyByteLimit?: number | undefined;
+
+  // Default 100 facts in one delivery.
+  readonly factLimit?: number | undefined;
+
+  // The `content-encoding` values the path takes besides none. Default none.
+  readonly encodings?: readonly 'gzip'[] | undefined;
+
+  // 'status' (default): 200 {"status":"stored","stored":<n>}, or 202 {"status":"ignored"}.
+  // 'empty': 200 {} for stored and for ignored, which an OTLP exporter expects.
+  readonly answer?: 'status' | 'empty' | undefined;
+
+  // A mark for the deployment's network rules. Default 'private'. It changes no answer.
+  readonly reach?: 'public' | 'private' | undefined;
+
+  // The path needs a proven sender: ingest answers 401 and stores nothing without one.
+  readonly sender?: SenderRequirement | undefined;
+
+  // Pure: it writes nothing.
+  readonly read: (delivery: Delivery, source: SourceContext) => Received;
+}
+
+// `hmac-sha256`: the header holds `sha256=<hex of HMAC-SHA256(secret, body as sent)>`. `token`: it
+// holds the secret, or `Bearer <secret>`. Neither signs a time, so a fact ID must come from the
+// signed body, and a replay is then a copy that every consumer drops.
+export interface DeliveryCheck {
+  readonly kind: 'hmac-sha256' | 'token';
+  readonly header: string;
+  readonly secretEnv: string;
+}
+
+// The proof a path needs, as data. `kind` picks the adapter of the `SenderProof` port. The audience
+// and the folder of trust documents belong to the deployment, so a plug-in names only the
+// variables that hold them.
+export interface SenderRequirement {
+  readonly kind: 'imp-identity';
+  readonly audienceEnv: string;
+  readonly issuersEnv: string;
+}
+
+export interface Delivery {
+  headers: Headers;
+
+  // after the check passed, and after decoding
+  body: Uint8Array;
+
+  // Present on every call for a path with a `sender`. `read` never takes a sender from `headers`.
+  sender?: Sender | undefined;
+}
+
+export interface Sender {
+  readonly issuer: string;
+  readonly subject: string;
+  readonly name: string;
+}
+
+// The port ingest calls for a path with a `sender`. It answers null when nothing proves a sender.
+export interface SenderProof {
+  prove: (requirement: SenderRequirement, headers: Headers) => Promise<Sender | null>;
+}
+
+export interface SourceContext {
+  name: string;
+  families: readonly FactFamily[];
+}
+
+// Ingest answers 401, 405, 413 and 415 itself, so `read` refuses with 400 or 422 only.
+export type Received =
+  | { kind: 'refused'; status: 400 | 422; reason: string }
+  | { kind: 'ignored' }
+  | { kind: 'batch'; batch: Batch };
+
+// Ingest reads the variables in `env` at start and passes their values.
+export interface PullPath {
+  readonly kind: 'pull';
+  readonly name: string;
+  readonly intervalMs: number | null;
+  readonly env?: readonly string[] | undefined;
+  readonly pull: (cursor: string | null, context: PullContext) => Promise<Pulled>;
+}
+
+export interface PullContext {
+  env: Readonly<Record<string, string>>;
+
+  // For a `resync` pull, the source's marked entities when the pull starts; empty for any other
+  // pull. Ingest removes each of these marks once the batch the pull returns is stored, so a pull
+  // that cannot read every marked entity answers `unavailable` and every mark stays.
+  marks: readonly ResyncMark[];
+}
+
+// An entity whose row may be behind the source, with the locator of the last fact ingest stored
+// for it, if that fact had one.
+export interface ResyncMark {
+  entity: EntityID;
+  locator: string | undefined;
+}
+
+export type Pulled =
+  | { kind: 'batch'; batch: Batch; cursor: string; isDone: boolean }
+  | { kind: 'expired' }
+  | { kind: 'unavailable'; retryAfterMs: number }
+  | { kind: 'rejected'; reason: string };
+
+export interface Batch {
+  facts: readonly PendingFact[];
+}
+
+// The body of a signed fact batch. Ingest builds each fact ID as `event:<source>:<event>`.
+export interface FactBatchBody {
+  v: 1;
+  facts: readonly {
+    type: string;
+    event: string;
+    entity: string;
+    actor?: string | undefined;
+    at: string;
+    payload: unknown;
+  }[];
+}
+
+export interface PendingFact {
+  stream: StreamName;
+  fact: FactEnvelope<unknown>;
+  blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
+
+  // What a source's resync pull needs to read the entity again, such as an API path. Ingest keeps
+  // it beside the entity's mark, and stores it in no fact.
+  locator?: string | undefined;
+}

package/src/store-reader/store-reader.ts

@@ -0,0 +1,76 @@
+import type { Kysely } from 'kysely';
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import type { StoreTables } from '../queries/types.ts';
+import type { Result } from '../types.ts';
+
+// One row of the `link` table that the host keeps in each consumer file.
+export interface LinkRow {
+  type: string;
+  from_id: string;
+  to_id: string;
+  scope: string | null;
+  origin: string;
+  confidence: number;
+  evidence: string;
+}
+
+// The two views of every connection. They cover the attached consumers only.
+export interface SharedViews {
+  // one row per entity of every entity view of every attached consumer
+  entity: {
+    id: string;
+    kind: string;
+    title: string;
+    url: string | null;
+    actor: string | null;
+    updated_at: string;
+  };
+
+  // every link of every attached consumer
+  link_effective: LinkRow;
+}
+
+// The host's own tables of a consumer file, reached as `<consumer>.<table>`.
+export type HostTables<Consumers extends readonly ConsumerDeclaration[]> = {
+  [Name in Consumers[number]['name'] as `${Name}.offsets`]: {
+    stream: string;
+    last_seq_num: number;
+  };
+};
+
+export type ConsumersDB<Consumers extends readonly ConsumerDeclaration[]> = StoreTables<Consumers> &
+  HostTables<Consumers> &
+  SharedViews;
+
+export interface AttachedCall {
+  // the version folder each attached consumer was read from, fixed for the whole call
+  attached: Readonly<Record<string, { version: string }>>;
+
+  // the `whenCurrent` consumers that had no current version
+  skipped: readonly string[];
+}
+
+// The work did not run: these consumers have no current version.
+export interface StoreReaderFailure {
+  reason: 'no-current-version';
+  consumers: readonly string[];
+}
+
+export interface WithConsumersOptions {
+  // More consumers to attach, each only when it has a current version. `db` has no typed table for
+  // them, and their rows show in the two shared views.
+  whenCurrent?: readonly ConsumerDeclaration[] | undefined;
+}
+
+// The port a query reads through. Adapters: Turso in the API, one in-memory database in the
+// testing package.
+export interface StoreReader {
+  // One read-only connection with the current file of each of `consumers` attached, and `work` in
+  // one transaction on it.
+  withConsumers: <const Consumers extends readonly ConsumerDeclaration[], Value>(
+    consumers: Consumers,
+    work: (db: Kysely<ConsumersDB<Consumers>>, call: AttachedCall) => Promise<Value>,
+    options?: WithConsumersOptions,
+  ) => Promise<Result<Value, StoreReaderFailure>>;
+  close: () => Promise<void>;
+}

package/src/tables/define-table.ts

@@ -0,0 +1,13 @@
+import type { ColumnDeclaration, IndexDeclaration, TableDeclaration } from '../types.ts';
+
+interface TableInput<Columns extends Readonly<Record<string, ColumnDeclaration>>> {
+  readonly columns: Columns;
+  readonly primaryKey: readonly NoInfer<keyof Columns & string>[];
+  readonly indexes: readonly IndexDeclaration<NoInfer<keyof Columns & string>>[];
+}
+
+export function defineTable<const Columns extends Readonly<Record<string, ColumnDeclaration>>>(
+  table: TableInput<Columns>,
+): TableDeclaration<Columns> {
+  return table;
+}

package/src/tables/table-row.ts

@@ -0,0 +1,20 @@
+import type { EntityID } from '../ids/types.ts';
+import type { ColumnDeclaration, TableDeclaration } from '../types.ts';
+
+interface KindTypes {
+  text: string;
+  integer: number;
+  real: number;
+  boolean: boolean;
+  json: unknown;
+  entityID: EntityID;
+  timestamp: string;
+}
+
+type ColumnValue<Column extends ColumnDeclaration> = Column['nullable'] extends true
+  ? KindTypes[Column['kind']] | null
+  : KindTypes[Column['kind']];
+
+export type TableRow<Table extends TableDeclaration> = {
+  -readonly [Name in keyof Table['columns']]: ColumnValue<Table['columns'][Name]>;
+};

package/src/types.ts

@@ -0,0 +1,47 @@
+import type { EntityID, FactID } from './ids/types.ts';
+
+export type Result<Value, Failure> = { ok: true; value: Value } | { ok: false; failure: Failure };
+
+export interface FactEnvelope<Payload> {
+  id: FactID;
+  type: string;
+  v: 1;
+  api: string;
+  entity: EntityID;
+  actor?: EntityID | undefined;
+  at: string;
+  cursor?: string | undefined;
+  payload: Payload;
+}
+
+// The name of a stream as a declaration writes it. A plain literal fits it, where the branded
+// `StreamName` needs a parse.
+export type DeclaredStreamName = `facts.${string}.${string}`;
+
+export type ColumnKind =
+  | 'text'
+  | 'integer'
+  | 'real'
+  | 'boolean'
+  | 'json'
+  | 'entityID'
+  | 'timestamp';
+
+export interface ColumnDeclaration {
+  readonly kind: ColumnKind;
+  readonly nullable?: boolean;
+}
+
+export interface IndexDeclaration<ColumnName extends string = string> {
+  readonly columns: readonly ColumnName[];
+}
+
+export interface TableDeclaration<
+  Columns extends Readonly<Record<string, ColumnDeclaration>> = Readonly<
+    Record<string, ColumnDeclaration>
+  >,
+> {
+  readonly columns: Columns;
+  readonly primaryKey: readonly string[];
+  readonly indexes: readonly IndexDeclaration[];
+}