@pepys/ingest 0.0.0-stage → 0.0.1

package/src/receive/make-receive-handler.ts

@@ -0,0 +1,241 @@
+import { createHash } from 'node:crypto';
+import { isDeliveryAuthentic } from '@pepys/kit';
+import type {
+  ReceivePath,
+  Received,
+  Sender,
+  SenderProof,
+  SenderRequirement,
+  SourceDefinition,
+} from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import { buildFailureAnswer, buildRetryAnswer } from './answers/build-failure-answer.ts';
+import { buildSuccessAnswer } from './answers/build-success-answer.ts';
+import { findBatchProblem } from './find-batch-problem.ts';
+import { planBatch } from './plan-batch.ts';
+import { readDecodedBody } from './read-decoded-body.ts';
+import { readSentBody } from './read-sent-body.ts';
+import type { RecentBodies } from './recent-bodies/create-recent-bodies.ts';
+import type { SourceMarks } from './resync-marks/create-resync-marks.ts';
+import { writeBatch } from './write-batch.ts';
+import type { StreamTurns } from './write-batch.ts';
+
+export interface ReceiveDependencies {
+  // the value of the path's `check.secretEnv`; an empty secret passes no delivery
+  secret: string;
+  senderProof: SenderProof;
+  append: BatchAppender;
+  turns: StreamTurns;
+  recentBodies: RecentBodies;
+
+  // null for a source with no resync pull, which reads no marks
+  marks: SourceMarks | null;
+}
+
+type Handler = (request: Request) => Promise<Response>;
+
+const defaultBodyByteLimit = 1_048_576;
+const defaultFactLimit = 100;
+
+// One delivery on the route of a receive path. Each step answers and stops, or goes on.
+export function makeReceiveHandler(
+  source: SourceDefinition,
+  path: ReceivePath,
+  dependencies: ReceiveDependencies,
+): Handler {
+  const byteLimit = path.bodyByteLimit ?? defaultBodyByteLimit;
+  const factLimit = path.factLimit ?? defaultFactLimit;
+
+  return async (request) => {
+    const checked = await readCheckedDelivery(request, path, dependencies);
+
+    if (checked.kind === 'answer') {
+      return checked.answer;
+    }
+
+    // The same answer as the first time, with no append, for a body this route stored in the last
+    // hour. A body is remembered only once it is stored, so a delivery that got 503 is never dropped.
+    const digest = createHash('sha256').update(checked.sent).digest('hex');
+    const key = buildRecentBodyKey(digest, checked.sender);
+    const known = dependencies.recentBodies.find(key);
+
+    if (known !== undefined) {
+      dependencies.marks?.noteCopy(digest, known.entities);
+
+      return buildSuccessAnswer(path, 'stored', known.stored);
+    }
+
+    const body = await readDecodedBody(
+      checked.sent,
+      request.headers.get('content-encoding'),
+      path.encodings ?? [],
+      byteLimit,
+    );
+
+    if (typeof body === 'string') {
+      return pickDecodeFailure(body);
+    }
+
+    const received = readSourceDelivery(source, path, {
+      headers: request.headers,
+      body,
+      ...(checked.sender === undefined ? {} : { sender: checked.sender }),
+    });
+
+    if (received === 'threw') {
+      return buildFailureAnswer(500, 'the source could not read the delivery');
+    }
+
+    if (received.kind === 'refused') {
+      return buildFailureAnswer(received.status, received.reason);
+    }
+
+    if (received.kind === 'ignored' || received.batch.facts.length === 0) {
+      return buildSuccessAnswer(path, 'ignored', 0);
+    }
+
+    const facts = received.batch.facts;
+
+    if (facts.length > factLimit) {
+      return buildFailureAnswer(413, `a batch holds ${factLimit} facts at most`);
+    }
+
+    const problem = findBatchProblem(source, received.batch);
+
+    if (problem !== undefined) {
+      return buildFailureAnswer(422, problem);
+    }
+
+    if (facts.some((pending) => pending.blobs.length > 0)) {
+      console.error(
+        `the source ${source.name} read a fact with a blob, and ingest has no blob store`,
+      );
+
+      return buildFailureAnswer(500, 'ingest has no blob store');
+    }
+
+    const plan = planBatch(received.batch);
+
+    if (!plan.ok) {
+      return buildFailureAnswer(413, 'one fact is larger than one append of the log takes');
+    }
+
+    const outcome = await writeBatch(plan.value, dependencies);
+
+    if (outcome === 'not-stored') {
+      return buildRetryAnswer();
+    }
+
+    if (outcome === 'refused') {
+      return buildFailureAnswer(500, 'the log refused the batch');
+    }
+
+    const entities = facts.map((pending) => pending.fact.entity);
+
+    dependencies.recentBodies.remember(key, { stored: facts.length, entities });
+
+    dependencies.marks?.noteStored(
+      digest,
+      facts.map((pending) => ({ entity: pending.fact.entity, at: pending.fact.at })),
+    );
+
+    return buildSuccessAnswer(path, 'stored', facts.length);
+  };
+}
+
+// A route of the path's `ignoredRoutes`: the same method, size, check and sender as the path, then
+// the ignored answer. It reads no body further and stores nothing.
+export function makeIgnoredRouteHandler(
+  path: ReceivePath,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+): Handler {
+  return async (request) => {
+    const checked = await readCheckedDelivery(request, path, dependencies);
+
+    return checked.kind === 'answer' ? checked.answer : buildSuccessAnswer(path, 'ignored', 0);
+  };
+}
+
+type CheckedDelivery =
+  | { kind: 'answer'; answer: Response }
+  | { kind: 'checked'; sent: Uint8Array; sender: Sender | undefined };
+
+// 405, 413, then 401 unless the check passes on the bytes as sent, then 401 unless the sender is
+// proven on a path that needs one. No code of the source has seen the delivery yet.
+async function readCheckedDelivery(
+  request: Request,
+  path: ReceivePath,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+): Promise<CheckedDelivery> {
+  if (request.method !== 'POST') {
+    return { kind: 'answer', answer: buildFailureAnswer(405, 'method not allowed') };
+  }
+
+  const sent = await readSentBody(request, path.bodyByteLimit ?? defaultBodyByteLimit);
+
+  if (sent === 'too-large') {
+    return { kind: 'answer', answer: buildFailureAnswer(413, 'body too large') };
+  }
+
+  if (!isDeliveryAuthentic(path.check, dependencies.secret, request.headers, sent)) {
+    return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
+  }
+
+  if (path.sender === undefined) {
+    return { kind: 'checked', sent, sender: undefined };
+  }
+
+  const sender = await readProvenSender(path.sender, request.headers, dependencies.senderProof);
+
+  if (sender === null) {
+    return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
+  }
+
+  return { kind: 'checked', sent, sender };
+}
+
+// A proof that throws proves nothing. The error may quote the request, so only its fact is logged.
+async function readProvenSender(
+  requirement: SenderRequirement,
+  headers: Headers,
+  senderProof: SenderProof,
+): Promise<Sender | null> {
+  try {
+    const sender = await senderProof.prove(requirement, headers);
+
+    return sender;
+  } catch {
+    console.error('the sender proof threw, so the delivery proves no sender');
+
+    return null;
+  }
+}
+
+// On a path with a sender, one body from two senders is two deliveries.
+function buildRecentBodyKey(digest: string, sender: Sender | undefined): string {
+  return sender === undefined ? digest : JSON.stringify([digest, sender.issuer, sender.subject]);
+}
+
+function pickDecodeFailure(failure: 'unsupported' | 'too-large' | 'corrupt'): Response {
+  return match(failure)
+    .with('unsupported', () => buildFailureAnswer(415, 'content encoding not accepted'))
+    .with('too-large', () => buildFailureAnswer(413, 'body too large'))
+    .with('corrupt', () => buildFailureAnswer(400, 'the body does not decode'))
+    .exhaustive();
+}
+
+// The source's own code. It is pure, so a throw is a fault of the deployment.
+function readSourceDelivery(
+  source: SourceDefinition,
+  path: ReceivePath,
+  delivery: Parameters<ReceivePath['read']>[0],
+): Received | 'threw' {
+  try {
+    return path.read(delivery, { name: source.name, families: source.families });
+  } catch (error: unknown) {
+    console.error(`the read of the source ${source.name} threw`, error);
+
+    return 'threw';
+  }
+}

package/src/receive/plan-batch.ts

@@ -0,0 +1,31 @@
+import type { Batch, FactEnvelope, Result, StreamName } from '@pepys/kit';
+import { planAppends } from '../fact-log/plan-appends.ts';
+
+export type BatchPlan = ReadonlyMap<StreamName, readonly (readonly FactEnvelope<unknown>[])[]>;
+
+// Groups the facts by stream, in batch order, and cuts each stream's facts into appends. One fact
+// that no append can hold refuses the whole batch, so a batch never stores in part for its content.
+export function planBatch(batch: Batch): Result<BatchPlan, { reason: 'too-large' }> {
+  const byStream = new Map<StreamName, FactEnvelope<unknown>[]>();
+
+  for (const pending of batch.facts) {
+    const facts = byStream.get(pending.stream) ?? [];
+
+    facts.push(pending.fact);
+    byStream.set(pending.stream, facts);
+  }
+
+  const plan = new Map<StreamName, readonly (readonly FactEnvelope<unknown>[])[]>();
+
+  for (const [stream, facts] of byStream) {
+    const runs = planAppends(facts);
+
+    if (!runs.ok) {
+      return { ok: false, failure: { reason: 'too-large' } };
+    }
+
+    plan.set(stream, runs.value);
+  }
+
+  return { ok: true, value: plan };
+}

package/src/receive/read-decoded-body.ts

@@ -0,0 +1,33 @@
+import { readStreamUpTo } from './read-stream-up-to.ts';
+
+export type DecodedBody = Uint8Array | 'unsupported' | 'too-large' | 'corrupt';
+
+// No `content-encoding`: the bytes as sent. `gzip` on a path that takes it: the bytes gunzipped as a
+// stream that stops at `limitBytes` decoded bytes, so a small body cannot unpack to a large one. Any
+// other value is `unsupported`, and a gzip body that does not gunzip is `corrupt`.
+export async function readDecodedBody(
+  sent: Uint8Array,
+  encoding: string | null,
+  accepted: readonly 'gzip'[],
+  limitBytes: number,
+): Promise<DecodedBody> {
+  if (encoding === null) {
+    return sent;
+  }
+
+  if (encoding.trim().toLowerCase() !== 'gzip' || !accepted.includes('gzip')) {
+    return 'unsupported';
+  }
+
+  // the copy is a Uint8Array over an ArrayBuffer, which the ESNext lib's BlobPart needs; a plugin's
+  // own compiler checks this file with its own lib
+  const decoded = new Blob([new Uint8Array(sent)])
+    .stream()
+    .pipeThrough(new DecompressionStream('gzip'));
+
+  try {
+    return await readStreamUpTo(decoded, limitBytes);
+  } catch {
+    return 'corrupt';
+  }
+}

package/src/receive/read-sent-body.ts

@@ -0,0 +1,22 @@
+import { readStreamUpTo } from './read-stream-up-to.ts';
+
+// The body as sent, up to `limitBytes`. A `content-length` over the limit refuses before any read,
+// and the count while reading refuses a body whose header understates it.
+export async function readSentBody(
+  request: Request,
+  limitBytes: number,
+): Promise<Uint8Array | 'too-large'> {
+  const declared = Number(request.headers.get('content-length') ?? '0');
+
+  if (declared > limitBytes) {
+    return 'too-large';
+  }
+
+  if (request.body === null) {
+    return new Uint8Array(0);
+  }
+
+  const body = await readStreamUpTo(request.body, limitBytes);
+
+  return body;
+}

package/src/receive/read-stream-up-to.ts

@@ -0,0 +1,44 @@
+// Reads a stream until it ends, or cancels it as soon as more than `limitBytes` have arrived, so a
+// body over the limit is never held whole.
+export async function readStreamUpTo(
+  stream: ReadableStream<Uint8Array>,
+  limitBytes: number,
+): Promise<Uint8Array | 'too-large'> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+
+  for (;;) {
+    const read = await reader.read();
+
+    if (read.done) {
+      break;
+    }
+
+    total += read.value.byteLength;
+
+    if (total > limitBytes) {
+      await reader.cancel();
+
+      return 'too-large';
+    }
+
+    chunks.push(read.value);
+  }
+
+  return joinChunks(chunks, total);
+}
+
+function joinChunks(chunks: readonly Uint8Array[], total: number): Uint8Array {
+  const joined = new Uint8Array(total);
+
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    joined.set(chunk, offset);
+
+    offset += chunk.byteLength;
+  }
+
+  return joined;
+}

package/src/receive/recent-bodies/create-recent-bodies.ts

@@ -0,0 +1,55 @@
+import type { EntityID } from '@pepys/kit';
+
+interface RecentBody {
+  stored: number;
+  entities: readonly EntityID[];
+}
+
+// The bodies one route stored in the last hour, by key. In memory only, so a restart forgets them.
+export interface RecentBodies {
+  find: (key: string) => RecentBody | undefined;
+  remember: (key: string, body: RecentBody) => void;
+}
+
+interface RecentBodiesOptions {
+  capacity: number;
+  maxAgeMs: number;
+  now: () => number;
+}
+
+interface Entry extends RecentBody {
+  atMs: number;
+}
+
+export function createRecentBodies(options: RecentBodiesOptions): RecentBodies {
+  // a Map keeps insertion order, and an entry is always set anew, so the oldest entry is first
+  const entries = new Map<string, Entry>();
+
+  const trimExpired = (): void => {
+    const oldestKept = options.now() - options.maxAgeMs;
+
+    for (const [key, entry] of entries) {
+      if (entry.atMs > oldestKept && entries.size <= options.capacity) {
+        return;
+      }
+
+      entries.delete(key);
+    }
+  };
+
+  return {
+    find: (key) => {
+      trimExpired();
+
+      const entry = entries.get(key);
+
+      return entry === undefined ? undefined : { stored: entry.stored, entities: entry.entities };
+    },
+    remember: (key, body) => {
+      entries.delete(key);
+      entries.set(key, { ...body, atMs: options.now() });
+
+      trimExpired();
+    },
+  };
+}

package/src/receive/resync-marks/collect-returned-entities.ts

@@ -0,0 +1,16 @@
+import type { EntityID } from '@pepys/kit';
+import type { LastBody } from './collect-tied-entities.ts';
+
+// A dropped copy of an earlier body may be a true return to an earlier state (A, B, A) when another
+// body was stored for one of its entities since.
+export function collectReturnedEntities(
+  lastBodies: ReadonlyMap<EntityID, LastBody>,
+  digest: string,
+  entities: readonly EntityID[],
+): EntityID[] {
+  return entities.filter((entity) => {
+    const last = lastBodies.get(entity);
+
+    return last !== undefined && last.digest !== digest;
+  });
+}

package/src/receive/resync-marks/collect-tied-entities.ts

@@ -0,0 +1,27 @@
+import type { EntityID } from '@pepys/kit';
+
+export interface LastBody {
+  digest: string;
+  at: string;
+}
+
+interface StoredFactTime {
+  entity: EntityID;
+  at: string;
+}
+
+// A new body ties with the last body stored for an entity when both have the same time and differ.
+// Two bodies with one time cannot be ordered by their facts, so the resync pull reads the entity.
+export function collectTiedEntities(
+  lastBodies: ReadonlyMap<EntityID, LastBody>,
+  digest: string,
+  facts: readonly StoredFactTime[],
+): EntityID[] {
+  return facts
+    .filter((fact) => {
+      const last = lastBodies.get(fact.entity);
+
+      return last !== undefined && last.at === fact.at && last.digest !== digest;
+    })
+    .map((fact) => fact.entity);
+}

package/src/receive/resync-marks/create-resync-marks.ts

@@ -0,0 +1,106 @@
+import type { EntityID } from '@pepys/kit';
+import { collectReturnedEntities } from './collect-returned-entities.ts';
+import { collectTiedEntities } from './collect-tied-entities.ts';
+import type { LastBody } from './collect-tied-entities.ts';
+
+// The entities whose row may be behind the source, which a source's resync pull reads again. The
+// pull reads the marks, and removes each one only after its fact is stored, so a failed pull keeps
+// it. In memory only, so a restart loses them.
+export interface ResyncMarks {
+  list: (source: string) => readonly EntityID[];
+  remove: (source: string, entity: EntityID) => void;
+}
+
+// What the receive route of one source tells the marks.
+export interface SourceMarks {
+  noteStored: (digest: string, facts: readonly { entity: EntityID; at: string }[]) => void;
+  noteCopy: (digest: string, entities: readonly EntityID[]) => void;
+}
+
+interface SourceState {
+  lastBodies: Map<EntityID, LastBody>;
+  marks: Set<EntityID>;
+}
+
+export interface ResyncMarkStore {
+  marks: ResyncMarks;
+  createSourceMarks: (source: string) => SourceMarks;
+}
+
+// `capacity` bounds the marks and the last bodies of each source; the oldest go first
+interface ResyncMarksOptions {
+  capacity: number;
+}
+
+export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore {
+  const sources = new Map<string, SourceState>();
+
+  const getState = (source: string): SourceState => {
+    const known = sources.get(source);
+
+    if (known !== undefined) {
+      return known;
+    }
+
+    const state: SourceState = { lastBodies: new Map(), marks: new Set() };
+
+    sources.set(source, state);
+
+    return state;
+  };
+
+  return {
+    marks: {
+      list: (source) => [...(sources.get(source)?.marks ?? [])],
+      remove: (source, entity) => {
+        sources.get(source)?.marks.delete(entity);
+      },
+    },
+    createSourceMarks: (source) => {
+      const state = getState(source);
+
+      return {
+        noteStored: (digest, facts) => {
+          addMarks(state.marks, collectTiedEntities(state.lastBodies, digest, facts), options);
+
+          for (const fact of facts) {
+            state.lastBodies.delete(fact.entity);
+            state.lastBodies.set(fact.entity, { digest, at: fact.at });
+          }
+
+          trimToCapacity(state.lastBodies, options.capacity);
+        },
+        noteCopy: (digest, entities) => {
+          addMarks(
+            state.marks,
+            collectReturnedEntities(state.lastBodies, digest, entities),
+            options,
+          );
+        },
+      };
+    },
+  };
+}
+
+function addMarks(
+  marks: Set<EntityID>,
+  entities: readonly EntityID[],
+  options: ResyncMarksOptions,
+): void {
+  for (const entity of entities) {
+    marks.add(entity);
+  }
+
+  trimToCapacity(marks, options.capacity);
+}
+
+// drops the oldest entries first: a Map and a Set keep insertion order
+function trimToCapacity(entries: Map<EntityID, LastBody> | Set<EntityID>, capacity: number): void {
+  for (const key of entries.keys()) {
+    if (entries.size <= capacity) {
+      return;
+    }
+
+    entries.delete(key);
+  }
+}

package/src/receive/write-batch.ts

@@ -0,0 +1,79 @@
+import type { StreamName } from '@pepys/kit';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import type { BatchPlan } from './plan-batch.ts';
+
+export type BatchOutcome = 'stored' | 'refused' | 'not-stored';
+
+// One chain of turns per stream, for the whole process. A delivery holds a stream's turn for all of
+// its appends on that stream, so the appends of two deliveries never interleave.
+export type StreamTurns = Map<StreamName, Promise<void>>;
+
+interface WriteBatchDependencies {
+  append: BatchAppender;
+  turns: StreamTurns;
+}
+
+// The streams append at the same time, and the appends of one stream one after the other. Every
+// stream settles before the answer, so a failure on one stream never leaves another in flight.
+export async function writeBatch(
+  plan: BatchPlan,
+  dependencies: WriteBatchDependencies,
+): Promise<BatchOutcome> {
+  const settled = await Promise.all(
+    [...plan].map(([stream, runs]) =>
+      withStreamTurn(dependencies.turns, stream, async () => {
+        for (const run of runs) {
+          const appended = await dependencies.append(stream, run);
+
+          if (!appended.ok) {
+            console.error(`an append to ${stream} failed`, appended.failure);
+
+            return appended.failure.kind;
+          }
+        }
+
+        return 'stored' as const;
+      }),
+    ),
+  );
+
+  return pickBatchOutcome(settled);
+}
+
+// A throw is a refusal the log gives no outcome for, such as a bad access token.
+async function withStreamTurn(
+  turns: StreamTurns,
+  stream: StreamName,
+  work: () => Promise<BatchOutcome>,
+): Promise<BatchOutcome> {
+  const previous = turns.get(stream) ?? Promise.resolve();
+  const done = Promise.withResolvers<void>();
+
+  turns.set(stream, done.promise);
+
+  try {
+    await previous;
+
+    return await work();
+  } catch (error: unknown) {
+    console.error(`an append to ${stream} threw`, error);
+
+    return 'refused';
+  } finally {
+    done.resolve();
+
+    if (turns.get(stream) === done.promise) {
+      turns.delete(stream);
+    }
+  }
+}
+
+// A refusal wins over `not-stored`: a repeat cannot mend it, so the sender must not be told to
+// repeat.
+function pickBatchOutcome(settled: readonly BatchOutcome[]): BatchOutcome {
+  if (settled.includes('refused')) {
+    return 'refused';
+  }
+
+  return settled.includes('not-stored') ? 'not-stored' : 'stored';
+}

package/src/sender-proof/fake/create-fake-sender-proof.ts

@@ -0,0 +1,14 @@
+import type { Sender, SenderProof } from '@pepys/kit';
+
+// The test stand-in for a verifier of signed identity tokens: the value of the `imp-identity`
+// header proves the sender it is mapped to, and any other value proves nothing.
+export function createFakeSenderProof(senders: Readonly<Record<string, Sender>>): SenderProof {
+  return {
+    prove: (_requirement, headers) => {
+      const token = headers.get('imp-identity');
+      const sender = token !== null && Object.hasOwn(senders, token) ? senders[token] : undefined;
+
+      return Promise.resolve(sender ?? null);
+    },
+  };
+}