@pepys/ingest 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,35 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/ingest</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/ingest"><img src="https://img.shields.io/npm/v/@pepys/ingest" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a>
+  </p>
+</div>
+
+pepys ingest, for Bun: it receives each source's deliveries and appends them to the fact log.
+
+A deployment serves one receive route for each receive path of its sources. Each route checks its
+delivery, and appends the facts it reads to the fact log.
+
+## Install
+
+```sh
+bun add @pepys/ingest
+```
+
+## Features
+
+- `serveIngest` and `createIngest`: the receive routes of a deployment.
+- The sender proof port, with the adapter that verifies a worker machine's identity token through
+  `@impz/client/identity`.
+- The fact log port, with its S2 and memory adapters.
+- The blob store port, with its Garage and local folder adapters.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of ingest and the log.

package/package.json

@@ -1,6 +1,35 @@
 {
   "name": "@pepys/ingest",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "pepys ingest: the receive routes, the fact log and the blob store",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/ingest"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils",
+    "!src/**/run-*-fixture.ts",
+    "!src/**/fixtures"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@impz/client": "6.12.0",
+    "@pepys/kit": "0.1.0",
+    "@pepys/schema": "0.1.0",
+    "@s2-dev/streamstore": "0.30.0",
+    "@zgeoff/std": "0.4.3",
+    "tiny-invariant": "1.3.3",
+    "ts-pattern": "5.9.2"
+  },
+  "devDependencies": {
+    "zod": "4.6.7"
+  }
+}

package/src/blob-store/blob-store.ts

@@ -0,0 +1,6 @@
+import type { BlobHash, Result } from '@pepys/kit';
+
+export interface BlobStore {
+  write: (bytes: Uint8Array) => Promise<Result<BlobHash, { reason: 'unavailable' }>>;
+  read: (hash: BlobHash) => Promise<Result<Uint8Array, { reason: 'missing' | 'unavailable' }>>;
+}

package/src/blob-store/build-blob-hash.ts

@@ -0,0 +1,6 @@
+import type { BlobHash } from '@pepys/kit';
+import { toTagged } from '@zgeoff/std';
+
+export function buildBlobHash(bytes: Uint8Array): BlobHash {
+  return toTagged<BlobHash>(new Bun.CryptoHasher('sha256').update(bytes).digest('hex'));
+}

package/src/blob-store/garage/create-garage-blob-store.ts

@@ -0,0 +1,82 @@
+import type { BlobHash, Result } from '@pepys/kit';
+import type { BlobStore } from '../blob-store.ts';
+import { buildBlobHash } from '../build-blob-hash.ts';
+
+interface GarageBlobStoreParameters {
+  endpoint: string;
+  region: string;
+  bucket: string;
+  accessKeyID: string;
+  secretAccessKey: string;
+  requestTimeoutMs: number;
+}
+
+export function createGarageBlobStore(parameters: GarageBlobStoreParameters): BlobStore {
+  const client = new Bun.S3Client({
+    endpoint: parameters.endpoint,
+    region: parameters.region,
+    bucket: parameters.bucket,
+    accessKeyId: parameters.accessKeyID,
+    secretAccessKey: parameters.secretAccessKey,
+  });
+
+  return {
+    write: (bytes) => writeBlob(client, bytes, parameters.requestTimeoutMs),
+    read: (hash) => readBlob(client, hash, parameters.requestTimeoutMs),
+  };
+}
+
+async function writeBlob(
+  client: Bun.S3Client,
+  bytes: Uint8Array,
+  timeoutMs: number,
+): Promise<Result<BlobHash, { reason: 'unavailable' }>> {
+  const hash = buildBlobHash(bytes);
+
+  try {
+    await withTimeout(timeoutMs, () => client.write(hash, bytes));
+  } catch {
+    return { ok: false, failure: { reason: 'unavailable' } };
+  }
+
+  return { ok: true, value: hash };
+}
+
+// The S3 client sets no request timeout of its own, so a store that never answers would hold the
+// caller forever.
+async function withTimeout<Value>(
+  timeoutMs: number,
+  request: () => Promise<Value>,
+): Promise<Value> {
+  const timeout = Promise.withResolvers<never>();
+
+  const timer = setTimeout(() => {
+    timeout.reject(new Error(`the request had no answer after ${timeoutMs} ms`));
+  }, timeoutMs);
+
+  try {
+    return await Promise.race([request(), timeout.promise]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function readBlob(
+  client: Bun.S3Client,
+  hash: BlobHash,
+  timeoutMs: number,
+): Promise<Result<Uint8Array, { reason: 'missing' | 'unavailable' }>> {
+  try {
+    return { ok: true, value: await withTimeout(timeoutMs, () => client.file(hash).bytes()) };
+  } catch (error: unknown) {
+    if (isMissingKey(error)) {
+      return { ok: false, failure: { reason: 'missing' } };
+    }
+
+    return { ok: false, failure: { reason: 'unavailable' } };
+  }
+}
+
+function isMissingKey(error: unknown): boolean {
+  return error instanceof Error && 'code' in error && error.code === 'NoSuchKey';
+}

package/src/blob-store/local-folder/create-local-folder-blob-store.ts

@@ -0,0 +1,56 @@
+import { rename, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { BlobHash, Result } from '@pepys/kit';
+import type { BlobStore } from '../blob-store.ts';
+import { buildBlobHash } from '../build-blob-hash.ts';
+
+export function createLocalFolderBlobStore(directory: string): BlobStore {
+  return {
+    write: (bytes) => writeBlob(directory, bytes),
+    read: (hash) => readBlob(directory, hash),
+  };
+}
+
+// A blob lands under its hash only once its bytes are complete, so a reader never sees part of one.
+async function writeBlob(
+  directory: string,
+  bytes: Uint8Array,
+): Promise<Result<BlobHash, { reason: 'unavailable' }>> {
+  const hash = buildBlobHash(bytes);
+  const partial = join(directory, `${hash}.${crypto.randomUUID()}.partial`);
+
+  try {
+    await writeFile(partial, bytes);
+    await rename(partial, join(directory, hash));
+  } catch {
+    await rm(partial, { force: true });
+
+    return { ok: false, failure: { reason: 'unavailable' } };
+  }
+
+  return { ok: true, value: hash };
+}
+
+const blobHashPattern = /^[0-9a-f]{64}$/u;
+
+async function readBlob(
+  directory: string,
+  hash: BlobHash,
+): Promise<Result<Uint8Array, { reason: 'missing' | 'unavailable' }>> {
+  // A hash that is not a SHA-256 in hex could name a path outside the folder.
+  if (!blobHashPattern.test(hash)) {
+    return { ok: false, failure: { reason: 'missing' } };
+  }
+
+  const file = Bun.file(join(directory, hash));
+
+  if (!(await file.exists())) {
+    return { ok: false, failure: { reason: 'missing' } };
+  }
+
+  try {
+    return { ok: true, value: await file.bytes() };
+  } catch {
+    return { ok: false, failure: { reason: 'unavailable' } };
+  }
+}

package/src/config/read-ingest-config.ts

@@ -0,0 +1,50 @@
+import type { Result } from '@pepys/kit';
+import { parsePort } from '@pepys/schema';
+
+export interface IngestConfig {
+  // the listen address; 127.0.0.1 unless the deployment sets another
+  host: string;
+  port: number;
+  s2: { endpoint: string; accessToken: string; basin: string };
+
+  // one value per name the deployment's paths read, each set and not empty
+  secrets: Readonly<Record<string, string>>;
+}
+
+const s2Names = ['PEPYS_S2_ENDPOINT', 'PEPYS_S2_ACCESS_TOKEN', 'PEPYS_S2_BASIN'] as const;
+
+// Reads every variable once and names each one that is missing, so ingest stops at start with the
+// whole list and never serves a route with an empty secret.
+export function readIngestConfig(
+  environment: Readonly<Record<string, string | undefined>>,
+  secretNames: readonly string[],
+): Result<IngestConfig, string[]> {
+  const required = [...s2Names, ...secretNames];
+  const missing = required.filter((name) => (environment[name] ?? '') === '');
+  const portText = environment['PEPYS_INGEST_PORT'] ?? '';
+  const port = portText === '' ? ({ ok: true, value: 8080 } as const) : parsePort(portText);
+  const host = environment['PEPYS_INGEST_HOST'] ?? '';
+
+  const problems = [
+    ...missing.map((name) => `${name} is not set`),
+    ...(port.ok ? [] : ['PEPYS_INGEST_PORT is not a port number']),
+  ];
+
+  if (!port.ok || problems.length > 0) {
+    return { ok: false, failure: problems };
+  }
+
+  return {
+    ok: true,
+    value: {
+      host: host === '' ? '127.0.0.1' : host,
+      port: port.value,
+      s2: {
+        endpoint: environment['PEPYS_S2_ENDPOINT'] ?? '',
+        accessToken: environment['PEPYS_S2_ACCESS_TOKEN'] ?? '',
+        basin: environment['PEPYS_S2_BASIN'] ?? '',
+      },
+      secrets: Object.fromEntries(secretNames.map((name) => [name, environment[name] ?? ''])),
+    },
+  };
+}

package/src/fact-log/check-append-batch.ts

@@ -0,0 +1,45 @@
+import type { FactEnvelope, Result } from '@pepys/kit';
+import { parseStoredBody } from './parse-stored-body.ts';
+
+// The limits of one s2-lite append. A record meters as its stored body in UTF-8 bytes plus 8, and
+// pepys records have no headers.
+export const appendLimits = { records: 1000, meteredBytes: 1_048_576 } as const;
+const recordOverheadBytes = 8;
+
+export type AppendBatchProblem =
+  | { reason: 'too-large' }
+  | { reason: 'invalid-envelope'; index: number };
+
+// The metered size of a stored body.
+export function toMeteredBytes(body: string): number {
+  return Buffer.byteLength(body, 'utf8') + recordOverheadBytes;
+}
+
+// Checks what an append needs before it reaches a stream, so every adapter refuses what s2-lite
+// refuses. An empty batch, or one over the record limit, is too large; then each fact must parse as
+// an envelope; then the batch must fit the byte limit. Returns the stored bodies, in order.
+export function checkAppendBatch(
+  facts: readonly FactEnvelope<unknown>[],
+): Result<readonly string[], AppendBatchProblem> {
+  if (facts.length === 0 || facts.length > appendLimits.records) {
+    return { ok: false, failure: { reason: 'too-large' } };
+  }
+
+  const bodies: string[] = [];
+
+  for (const [index, fact] of facts.entries()) {
+    const body = parseStoredBody(fact);
+
+    if (body === null) {
+      return { ok: false, failure: { reason: 'invalid-envelope', index } };
+    }
+
+    bodies.push(body);
+  }
+
+  const meteredBytes = bodies.reduce((sum, body) => sum + toMeteredBytes(body), 0);
+
+  return meteredBytes > appendLimits.meteredBytes
+    ? { ok: false, failure: { reason: 'too-large' } }
+    : { ok: true, value: bodies };
+}

package/src/fact-log/fact-log.ts

@@ -0,0 +1,35 @@
+import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+
+// Fact `i` of an acknowledged batch has the offset `firstOffset + i`.
+export type AppendBatchOutcome =
+  | { kind: 'acknowledged'; firstOffset: StreamOffset; count: number }
+
+  // Nothing was stored.
+  | { kind: 'rejected'; reason: 'tail-mismatch' | 'unknown-stream' | 'too-large' }
+  | { kind: 'rejected'; reason: 'invalid-envelope'; index: number }
+
+  // The reply was lost: the batch is stored whole, or not at all.
+  | { kind: 'unknown' };
+
+export interface StoredFact {
+  offset: StreamOffset;
+  fact: FactEnvelope<unknown>;
+  body: string;
+  appendedAtMs: number;
+}
+
+export interface FactLog {
+  // Stores every fact, in order, in one append that the log refuses unless the stream's tail is
+  // `expectedTail`. All the facts land or none does.
+  appendBatch: (
+    stream: StreamName,
+    facts: readonly FactEnvelope<unknown>[],
+    expectedTail: StreamOffset,
+  ) => Promise<AppendBatchOutcome>;
+  read: (
+    stream: StreamName,
+    from: StreamOffset,
+    count: number,
+  ) => Promise<Result<readonly StoredFact[], { reason: 'unknown-stream' }>>;
+  readTail: (stream: StreamName) => Promise<Result<StreamOffset, { reason: 'unknown-stream' }>>;
+}

package/src/fact-log/make-batch-appender.ts

@@ -0,0 +1,198 @@
+import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+import { toTagged } from '@zgeoff/std';
+import { P, match } from 'ts-pattern';
+import type { FactLog, StoredFact } from './fact-log.ts';
+import { pickSettlement } from './pick-settlement.ts';
+
+interface BatchAppenderOptions {
+  log: FactLog;
+  maxAttempts: number;
+  retryDelayMs: number;
+}
+
+// `refused`: the log gave a reason that no retry changes. `not-stored`: no acknowledgement after
+// `attempts` attempts, so the facts may or may not be in the log.
+export type AppendFailure =
+  | { kind: 'refused'; reason: 'unknown-stream' | 'invalid-envelope' | 'too-large' }
+  | { kind: 'not-stored'; attempts: number };
+
+export type BatchAppender = (
+  stream: StreamName,
+  facts: readonly FactEnvelope<unknown>[],
+) => Promise<Result<{ firstOffset: StreamOffset; count: number }, AppendFailure>>;
+
+// Owns the retries of one append. The guard goes on every attempt, the first included. After an
+// attempt whose outcome is unknown, the appender reads the offsets that the attempt aimed at before
+// it sends another, so the run is stored once.
+export function makeBatchAppender(options: BatchAppenderOptions): BatchAppender {
+  return async (stream, facts) => {
+    let pendingTail: StreamOffset | null = null;
+
+    for (let attempt = 1; ; attempt += 1) {
+      const result = await runAttempt(options, stream, facts, pendingTail);
+
+      if (result.kind === 'stored') {
+        return { ok: true, value: result.stored };
+      }
+
+      if (result.kind === 'refused') {
+        return { ok: false, failure: result.failure };
+      }
+
+      pendingTail = result.pendingTail;
+
+      if (attempt >= options.maxAttempts) {
+        return { ok: false, failure: { kind: 'not-stored', attempts: attempt } };
+      }
+
+      await Bun.sleep(options.retryDelayMs * attempt);
+    }
+  };
+}
+
+interface Stored {
+  firstOffset: StreamOffset;
+  count: number;
+}
+
+type StopFailure = Extract<AppendFailure, { kind: 'refused' }>;
+
+// What one attempt leaves: the answer, or the tail that earlier attempts with an unknown outcome
+// aimed at, which a later attempt can still land on. `null` when no such attempt is open.
+type AttemptResult =
+  | { kind: 'stored'; stored: Stored }
+  | { kind: 'refused'; failure: StopFailure }
+  | { kind: 'again'; pendingTail: StreamOffset | null };
+
+async function runAttempt(
+  options: BatchAppenderOptions,
+  stream: StreamName,
+  facts: readonly FactEnvelope<unknown>[],
+  pendingTail: StreamOffset | null,
+): Promise<AttemptResult> {
+  // Every attempt that is still open aimed at the same tail: a new tail comes only after the
+  // offsets there are known to hold another writer's records.
+  const read =
+    pendingTail === null
+      ? await readTail(options.log, stream)
+      : { ok: true as const, value: pendingTail };
+
+  if (read === null) {
+    return { kind: 'again', pendingTail };
+  }
+
+  if (!read.ok) {
+    return { kind: 'refused', failure: read.failure };
+  }
+
+  const aimed = read.value;
+
+  const outcome = await options.log.appendBatch(stream, facts, aimed);
+
+  return match(outcome)
+    .with({ kind: 'acknowledged' }, (acknowledged) => ({
+      kind: 'stored' as const,
+      stored: { firstOffset: acknowledged.firstOffset, count: acknowledged.count },
+    }))
+    .with({ kind: 'rejected', reason: 'tail-mismatch' }, () =>
+      pendingTail === null
+        ? { kind: 'again' as const, pendingTail }
+        : settleAttempt(options, stream, facts, aimed),
+    )
+    .with(
+      { kind: 'rejected', reason: P.union('unknown-stream', 'invalid-envelope', 'too-large') },
+      (rejected) => ({
+        kind: 'refused' as const,
+        failure: { kind: 'refused' as const, reason: rejected.reason },
+      }),
+    )
+    .with({ kind: 'unknown' }, () => settleAttempt(options, stream, facts, aimed))
+    .exhaustive();
+}
+
+// A failed read stored nothing, so the next attempt repeats it. Returns `null` for a failed read,
+// and the failure for a stream the log lacks. An error the append throws is a refusal the log gives
+// no outcome for, such as a bad token, and a retry cannot change it.
+async function readTail(
+  log: FactLog,
+  stream: StreamName,
+): Promise<Result<StreamOffset, StopFailure> | null> {
+  try {
+    const tail = await log.readTail(stream);
+
+    return tail.ok ? tail : { ok: false, failure: { kind: 'refused', reason: 'unknown-stream' } };
+  } catch (error: unknown) {
+    console.error('a read before an append failed', error);
+
+    return null;
+  }
+}
+
+// Reads exactly the offsets `[tail, tail + n)` that an attempt aimed at, and compares their fact IDs
+// with the run's, in order. A search for the first ID anywhere after the tail would take another
+// writer's copy of it for the whole run.
+async function settleAttempt(
+  options: BatchAppenderOptions,
+  stream: StreamName,
+  facts: readonly FactEnvelope<unknown>[],
+  tail: StreamOffset,
+): Promise<AttemptResult> {
+  const stored = await readRun(options.log, stream, tail, facts.length).catch((error: unknown) => {
+    console.error('a read that settles an append failed', error);
+
+    return null;
+  });
+
+  if (stored === null) {
+    return { kind: 'again', pendingTail: tail };
+  }
+
+  if (!stored.ok) {
+    return { kind: 'refused', failure: { kind: 'refused', reason: 'unknown-stream' } };
+  }
+
+  return match(
+    pickSettlement(
+      facts.map((fact) => fact.id),
+      stored.value,
+    ),
+  )
+    .with('landed', () => ({
+      kind: 'stored' as const,
+      stored: { firstOffset: tail, count: facts.length },
+    }))
+    .with('not-landed', () => ({ kind: 'again' as const, pendingTail: tail }))
+    .with('taken', () => ({ kind: 'again' as const, pendingTail: null }))
+    .exhaustive();
+}
+
+// A read can end before `count` records, so it repeats from the next offset until it has them all
+// or the stream ends.
+async function readRun(
+  log: FactLog,
+  stream: StreamName,
+  from: StreamOffset,
+  count: number,
+): Promise<Result<readonly StoredFact[], { reason: 'unknown-stream' }>> {
+  const records: StoredFact[] = [];
+
+  while (records.length < count) {
+    const read = await log.read(
+      stream,
+      toTagged<StreamOffset>(from + records.length),
+      count - records.length,
+    );
+
+    if (!read.ok) {
+      return read;
+    }
+
+    if (read.value.length === 0) {
+      break;
+    }
+
+    records.push(...read.value);
+  }
+
+  return { ok: true, value: records };
+}