@pepys/ingest 0.0.0-stage → 0.1.0

package/src/receive/create-ingest.ts

@@ -0,0 +1,103 @@
+import type { CheckedDeployment, SenderProof } from '@pepys/kit';
+import type { FactLog } from '../fact-log/fact-log.ts';
+import { makeBatchAppender } from '../fact-log/make-batch-appender.ts';
+import { makePullRunner } from '../pull/make-pull-runner.ts';
+import type { PullRunner } from '../pull/make-pull-runner.ts';
+import { buildFailureAnswer } from './answers/build-failure-answer.ts';
+import { makeIgnoredRouteHandler, makeReceiveHandler } from './make-receive-handler.ts';
+import { createRecentBodies } from './recent-bodies/create-recent-bodies.ts';
+import { createResyncMarks } from './resync-marks/create-resync-marks.ts';
+import type { ResyncMarks } from './resync-marks/create-resync-marks.ts';
+import type { StreamTurns } from './write-batch.ts';
+
+export interface IngestOptions {
+  deployment: CheckedDeployment;
+
+  // One value per `check.secretEnv` and per name in a pull path's `env`. A missing or empty secret
+  // refuses every delivery.
+  secrets: Readonly<Record<string, string>>;
+  log: FactLog;
+  senderProof: SenderProof;
+  appendRetry: { maxAttempts: number; retryDelayMs: number };
+  now: () => number;
+}
+
+export interface Ingest {
+  // every receive route of the deployment; any other path answers 404
+  handleRequest: (request: Request) => Promise<Response>;
+
+  // what a source's resync pull reads
+  resyncMarks: ResyncMarks;
+
+  // one run of a pull path of the deployment, with its batch stored
+  runPull: PullRunner;
+}
+
+const recentBodyCapacity = 10_000;
+const recentBodyMaxAgeMs = 3_600_000;
+
+// The receive routes of a deployment over a given log, as one fetch function with no listener.
+export function createIngest(options: IngestOptions): Ingest {
+  const append = makeBatchAppender({ log: options.log, ...options.appendRetry });
+
+  const turns: StreamTurns = new Map();
+
+  const resync = createResyncMarks({ capacity: recentBodyCapacity });
+
+  const handlers = new Map<string, (request: Request) => Promise<Response>>();
+
+  for (const source of options.deployment.sources) {
+    const path = source.live;
+
+    if (path?.kind !== 'receive') {
+      continue;
+    }
+
+    const secret = options.secrets[path.check.secretEnv] ?? '';
+
+    handlers.set(
+      path.route,
+      makeReceiveHandler(source, path, {
+        secret,
+        senderProof: options.senderProof,
+        append,
+        turns,
+        recentBodies: createRecentBodies({
+          capacity: recentBodyCapacity,
+          maxAgeMs: recentBodyMaxAgeMs,
+          now: options.now,
+        }),
+        marks: source.resync === undefined ? null : resync.createSourceMarks(source.name),
+      }),
+    );
+
+    for (const route of path.ignoredRoutes ?? []) {
+      handlers.set(
+        route,
+        makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof }),
+      );
+    }
+  }
+
+  return {
+    handleRequest: async (request) => {
+      const handler = handlers.get(new URL(request.url).pathname);
+
+      if (handler === undefined) {
+        return buildFailureAnswer(404, 'not found');
+      }
+
+      const answer = await handler(request);
+
+      return answer;
+    },
+    resyncMarks: resync.marks,
+    runPull: makePullRunner({
+      deployment: options.deployment,
+      secrets: options.secrets,
+      append,
+      turns,
+      resyncMarks: resync.marks,
+    }),
+  };
+}

package/src/receive/find-batch-problem.ts

@@ -0,0 +1,26 @@
+import { parseFactEnvelope } from '@pepys/kit';
+import type { Batch, SourceContext } from '@pepys/kit';
+
+// Every fact is checked before any is written: its stream must be one of the source's families, and
+// its envelope must parse.
+export function findBatchProblem(source: SourceContext, batch: Batch): string | undefined {
+  const streams = new Set<string>(source.families.map((family) => family.stream));
+
+  for (const [index, pending] of batch.facts.entries()) {
+    if (!streams.has(pending.stream)) {
+      return `fact ${index}: the source ${source.name} has no stream ${pending.stream}`;
+    }
+
+    const envelope = parseFactEnvelope(pending.fact);
+
+    if (!envelope.ok) {
+      const issues = envelope.failure.issues.map(
+        (issue) => `${issue.path.join('.')}: ${issue.message}`,
+      );
+
+      return `fact ${index} is not a valid envelope: ${issues.join('; ')}`;
+    }
+  }
+
+  return undefined;
+}

package/src/receive/make-receive-handler.ts

@@ -0,0 +1,245 @@
+import { createHash } from 'node:crypto';
+import { isDeliveryAuthentic } from '@pepys/kit';
+import type {
+  ReceivePath,
+  Received,
+  Sender,
+  SenderProof,
+  SenderRequirement,
+  SourceDefinition,
+} from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import { buildFailureAnswer, buildRetryAnswer } from './answers/build-failure-answer.ts';
+import { buildSuccessAnswer } from './answers/build-success-answer.ts';
+import { findBatchProblem } from './find-batch-problem.ts';
+import { planBatch } from './plan-batch.ts';
+import { readDecodedBody } from './read-decoded-body.ts';
+import { readSentBody } from './read-sent-body.ts';
+import type { RecentBodies } from './recent-bodies/create-recent-bodies.ts';
+import type { SourceMarks } from './resync-marks/create-resync-marks.ts';
+import { writeBatch } from './write-batch.ts';
+import type { StreamTurns } from './write-batch.ts';
+
+export interface ReceiveDependencies {
+  // the value of the path's `check.secretEnv`; an empty secret passes no delivery
+  secret: string;
+  senderProof: SenderProof;
+  append: BatchAppender;
+  turns: StreamTurns;
+  recentBodies: RecentBodies;
+
+  // null for a source with no resync pull, which reads no marks
+  marks: SourceMarks | null;
+}
+
+type Handler = (request: Request) => Promise<Response>;
+
+const defaultBodyByteLimit = 1_048_576;
+const defaultFactLimit = 100;
+
+// One delivery on the route of a receive path. Each step answers and stops, or goes on.
+export function makeReceiveHandler(
+  source: SourceDefinition,
+  path: ReceivePath,
+  dependencies: ReceiveDependencies,
+): Handler {
+  const byteLimit = path.bodyByteLimit ?? defaultBodyByteLimit;
+  const factLimit = path.factLimit ?? defaultFactLimit;
+
+  return async (request) => {
+    const checked = await readCheckedDelivery(request, path, dependencies);
+
+    if (checked.kind === 'answer') {
+      return checked.answer;
+    }
+
+    // The same answer as the first time, with no append, for a body this route stored in the last
+    // hour. A body is remembered only once it is stored, so a delivery that got 503 is never dropped.
+    const digest = createHash('sha256').update(checked.sent).digest('hex');
+    const key = buildRecentBodyKey(digest, checked.sender);
+    const known = dependencies.recentBodies.find(key);
+
+    if (known !== undefined) {
+      dependencies.marks?.noteCopy(digest, known.entities);
+
+      return buildSuccessAnswer(path, 'stored', known.stored);
+    }
+
+    const body = await readDecodedBody(
+      checked.sent,
+      request.headers.get('content-encoding'),
+      path.encodings ?? [],
+      byteLimit,
+    );
+
+    if (typeof body === 'string') {
+      return pickDecodeFailure(body);
+    }
+
+    const received = readSourceDelivery(source, path, {
+      headers: request.headers,
+      body,
+      ...(checked.sender === undefined ? {} : { sender: checked.sender }),
+    });
+
+    if (received === 'threw') {
+      return buildFailureAnswer(500, 'the source could not read the delivery');
+    }
+
+    if (received.kind === 'refused') {
+      return buildFailureAnswer(received.status, received.reason);
+    }
+
+    if (received.kind === 'ignored' || received.batch.facts.length === 0) {
+      return buildSuccessAnswer(path, 'ignored', 0);
+    }
+
+    const facts = received.batch.facts;
+
+    if (facts.length > factLimit) {
+      return buildFailureAnswer(413, `a batch holds ${factLimit} facts at most`);
+    }
+
+    const problem = findBatchProblem(source, received.batch);
+
+    if (problem !== undefined) {
+      return buildFailureAnswer(422, problem);
+    }
+
+    if (facts.some((pending) => pending.blobs.length > 0)) {
+      console.error(
+        `the source ${source.name} read a fact with a blob, and ingest has no blob store`,
+      );
+
+      return buildFailureAnswer(500, 'ingest has no blob store');
+    }
+
+    const plan = planBatch(received.batch);
+
+    if (!plan.ok) {
+      return buildFailureAnswer(413, 'one fact is larger than one append of the log takes');
+    }
+
+    const outcome = await writeBatch(plan.value, dependencies);
+
+    if (outcome === 'not-stored') {
+      return buildRetryAnswer();
+    }
+
+    if (outcome === 'refused') {
+      return buildFailureAnswer(500, 'the log refused the batch');
+    }
+
+    const entities = facts.map((pending) => pending.fact.entity);
+
+    dependencies.recentBodies.remember(key, { stored: facts.length, entities });
+
+    dependencies.marks?.noteStored(
+      digest,
+      facts.map((pending) => ({
+        entity: pending.fact.entity,
+        at: pending.fact.at,
+        locator: pending.locator,
+      })),
+    );
+
+    return buildSuccessAnswer(path, 'stored', facts.length);
+  };
+}
+
+// A route of the path's `ignoredRoutes`: the same method, size, check and sender as the path, then
+// the ignored answer. It reads no body further and stores nothing.
+export function makeIgnoredRouteHandler(
+  path: ReceivePath,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+): Handler {
+  return async (request) => {
+    const checked = await readCheckedDelivery(request, path, dependencies);
+
+    return checked.kind === 'answer' ? checked.answer : buildSuccessAnswer(path, 'ignored', 0);
+  };
+}
+
+type CheckedDelivery =
+  | { kind: 'answer'; answer: Response }
+  | { kind: 'checked'; sent: Uint8Array; sender: Sender | undefined };
+
+// 405, 413, then 401 unless the check passes on the bytes as sent, then 401 unless the sender is
+// proven on a path that needs one. No code of the source has seen the delivery yet.
+async function readCheckedDelivery(
+  request: Request,
+  path: ReceivePath,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+): Promise<CheckedDelivery> {
+  if (request.method !== 'POST') {
+    return { kind: 'answer', answer: buildFailureAnswer(405, 'method not allowed') };
+  }
+
+  const sent = await readSentBody(request, path.bodyByteLimit ?? defaultBodyByteLimit);
+
+  if (sent === 'too-large') {
+    return { kind: 'answer', answer: buildFailureAnswer(413, 'body too large') };
+  }
+
+  if (!isDeliveryAuthentic(path.check, dependencies.secret, request.headers, sent)) {
+    return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
+  }
+
+  if (path.sender === undefined) {
+    return { kind: 'checked', sent, sender: undefined };
+  }
+
+  const sender = await readProvenSender(path.sender, request.headers, dependencies.senderProof);
+
+  if (sender === null) {
+    return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
+  }
+
+  return { kind: 'checked', sent, sender };
+}
+
+// A proof that throws proves nothing. The error may quote the request, so only its fact is logged.
+async function readProvenSender(
+  requirement: SenderRequirement,
+  headers: Headers,
+  senderProof: SenderProof,
+): Promise<Sender | null> {
+  try {
+    const sender = await senderProof.prove(requirement, headers);
+
+    return sender;
+  } catch {
+    console.error('the sender proof threw, so the delivery proves no sender');
+
+    return null;
+  }
+}
+
+// On a path with a sender, one body from two senders is two deliveries.
+function buildRecentBodyKey(digest: string, sender: Sender | undefined): string {
+  return sender === undefined ? digest : JSON.stringify([digest, sender.issuer, sender.subject]);
+}
+
+function pickDecodeFailure(failure: 'unsupported' | 'too-large' | 'corrupt'): Response {
+  return match(failure)
+    .with('unsupported', () => buildFailureAnswer(415, 'content encoding not accepted'))
+    .with('too-large', () => buildFailureAnswer(413, 'body too large'))
+    .with('corrupt', () => buildFailureAnswer(400, 'the body does not decode'))
+    .exhaustive();
+}
+
+// The source's own code. It is pure, so a throw is a fault of the deployment.
+function readSourceDelivery(
+  source: SourceDefinition,
+  path: ReceivePath,
+  delivery: Parameters<ReceivePath['read']>[0],
+): Received | 'threw' {
+  try {
+    return path.read(delivery, { name: source.name, families: source.families });
+  } catch (error: unknown) {
+    console.error(`the read of the source ${source.name} threw`, error);
+
+    return 'threw';
+  }
+}

package/src/receive/plan-batch.ts

@@ -0,0 +1,31 @@
+import type { Batch, FactEnvelope, Result, StreamName } from '@pepys/kit';
+import { planAppends } from '../fact-log/plan-appends.ts';
+
+export type BatchPlan = ReadonlyMap<StreamName, readonly (readonly FactEnvelope<unknown>[])[]>;
+
+// Groups the facts by stream, in batch order, and cuts each stream's facts into appends. One fact
+// that no append can hold refuses the whole batch, so a batch never stores in part for its content.
+export function planBatch(batch: Batch): Result<BatchPlan, { reason: 'too-large' }> {
+  const byStream = new Map<StreamName, FactEnvelope<unknown>[]>();
+
+  for (const pending of batch.facts) {
+    const facts = byStream.get(pending.stream) ?? [];
+
+    facts.push(pending.fact);
+    byStream.set(pending.stream, facts);
+  }
+
+  const plan = new Map<StreamName, readonly (readonly FactEnvelope<unknown>[])[]>();
+
+  for (const [stream, facts] of byStream) {
+    const runs = planAppends(facts);
+
+    if (!runs.ok) {
+      return { ok: false, failure: { reason: 'too-large' } };
+    }
+
+    plan.set(stream, runs.value);
+  }
+
+  return { ok: true, value: plan };
+}

package/src/receive/read-decoded-body.ts

@@ -0,0 +1,33 @@
+import { readStreamUpTo } from './read-stream-up-to.ts';
+
+export type DecodedBody = Uint8Array | 'unsupported' | 'too-large' | 'corrupt';
+
+// No `content-encoding`: the bytes as sent. `gzip` on a path that takes it: the bytes gunzipped as a
+// stream that stops at `limitBytes` decoded bytes, so a small body cannot unpack to a large one. Any
+// other value is `unsupported`, and a gzip body that does not gunzip is `corrupt`.
+export async function readDecodedBody(
+  sent: Uint8Array,
+  encoding: string | null,
+  accepted: readonly 'gzip'[],
+  limitBytes: number,
+): Promise<DecodedBody> {
+  if (encoding === null) {
+    return sent;
+  }
+
+  if (encoding.trim().toLowerCase() !== 'gzip' || !accepted.includes('gzip')) {
+    return 'unsupported';
+  }
+
+  // the copy is a Uint8Array over an ArrayBuffer, which the ESNext lib's BlobPart needs; a plugin's
+  // own compiler checks this file with its own lib
+  const decoded = new Blob([new Uint8Array(sent)])
+    .stream()
+    .pipeThrough(new DecompressionStream('gzip'));
+
+  try {
+    return await readStreamUpTo(decoded, limitBytes);
+  } catch {
+    return 'corrupt';
+  }
+}

package/src/receive/read-sent-body.ts

@@ -0,0 +1,22 @@
+import { readStreamUpTo } from './read-stream-up-to.ts';
+
+// The body as sent, up to `limitBytes`. A `content-length` over the limit refuses before any read,
+// and the count while reading refuses a body whose header understates it.
+export async function readSentBody(
+  request: Request,
+  limitBytes: number,
+): Promise<Uint8Array | 'too-large'> {
+  const declared = Number(request.headers.get('content-length') ?? '0');
+
+  if (declared > limitBytes) {
+    return 'too-large';
+  }
+
+  if (request.body === null) {
+    return new Uint8Array(0);
+  }
+
+  const body = await readStreamUpTo(request.body, limitBytes);
+
+  return body;
+}

package/src/receive/read-stream-up-to.ts

@@ -0,0 +1,44 @@
+// Reads a stream until it ends, or cancels it as soon as more than `limitBytes` have arrived, so a
+// body over the limit is never held whole.
+export async function readStreamUpTo(
+  stream: ReadableStream<Uint8Array>,
+  limitBytes: number,
+): Promise<Uint8Array | 'too-large'> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+
+  for (;;) {
+    const read = await reader.read();
+
+    if (read.done) {
+      break;
+    }
+
+    total += read.value.byteLength;
+
+    if (total > limitBytes) {
+      await reader.cancel();
+
+      return 'too-large';
+    }
+
+    chunks.push(read.value);
+  }
+
+  return joinChunks(chunks, total);
+}
+
+function joinChunks(chunks: readonly Uint8Array[], total: number): Uint8Array {
+  const joined = new Uint8Array(total);
+
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    joined.set(chunk, offset);
+
+    offset += chunk.byteLength;
+  }
+
+  return joined;
+}

package/src/receive/recent-bodies/create-recent-bodies.ts

@@ -0,0 +1,55 @@
+import type { EntityID } from '@pepys/kit';
+
+interface RecentBody {
+  stored: number;
+  entities: readonly EntityID[];
+}
+
+// The bodies one route stored in the last hour, by key. In memory only, so a restart forgets them.
+export interface RecentBodies {
+  find: (key: string) => RecentBody | undefined;
+  remember: (key: string, body: RecentBody) => void;
+}
+
+interface RecentBodiesOptions {
+  capacity: number;
+  maxAgeMs: number;
+  now: () => number;
+}
+
+interface Entry extends RecentBody {
+  atMs: number;
+}
+
+export function createRecentBodies(options: RecentBodiesOptions): RecentBodies {
+  // a Map keeps insertion order, and an entry is always set anew, so the oldest entry is first
+  const entries = new Map<string, Entry>();
+
+  const trimExpired = (): void => {
+    const oldestKept = options.now() - options.maxAgeMs;
+
+    for (const [key, entry] of entries) {
+      if (entry.atMs > oldestKept && entries.size <= options.capacity) {
+        return;
+      }
+
+      entries.delete(key);
+    }
+  };
+
+  return {
+    find: (key) => {
+      trimExpired();
+
+      const entry = entries.get(key);
+
+      return entry === undefined ? undefined : { stored: entry.stored, entities: entry.entities };
+    },
+    remember: (key, body) => {
+      entries.delete(key);
+      entries.set(key, { ...body, atMs: options.now() });
+
+      trimExpired();
+    },
+  };
+}

package/src/receive/resync-marks/collect-returned-entities.ts

@@ -0,0 +1,16 @@
+import type { EntityID } from '@pepys/kit';
+import type { LastBody } from './collect-tied-entities.ts';
+
+// A dropped copy of an earlier body may be a true return to an earlier state (A, B, A) when another
+// body was stored for one of its entities since.
+export function collectReturnedEntities(
+  lastBodies: ReadonlyMap<EntityID, LastBody>,
+  digest: string,
+  entities: readonly EntityID[],
+): EntityID[] {
+  return entities.filter((entity) => {
+    const last = lastBodies.get(entity);
+
+    return last !== undefined && last.digest !== digest;
+  });
+}

package/src/receive/resync-marks/collect-tied-entities.ts

@@ -0,0 +1,29 @@
+import type { EntityID } from '@pepys/kit';
+
+export interface LastBody {
+  digest: string;
+  at: string;
+  locator: string | undefined;
+}
+
+export interface StoredFactTime {
+  entity: EntityID;
+  at: string;
+  locator: string | undefined;
+}
+
+// A new body ties with the last body stored for an entity when both have the same time and differ.
+// Two bodies with one time cannot be ordered by their facts, so the resync pull reads the entity.
+export function collectTiedEntities(
+  lastBodies: ReadonlyMap<EntityID, LastBody>,
+  digest: string,
+  facts: readonly StoredFactTime[],
+): EntityID[] {
+  return facts
+    .filter((fact) => {
+      const last = lastBodies.get(fact.entity);
+
+      return last !== undefined && last.at === fact.at && last.digest !== digest;
+    })
+    .map((fact) => fact.entity);
+}