@pepys/ingest 0.0.0-stage → 0.0.1

package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts

@@ -0,0 +1,58 @@
+import type { Sender, SenderProof } from '@pepys/kit';
+import type { ImpIdentityTrust, ImpIdentityVerifier, VerifiedImp } from './types.ts';
+
+interface ImpIdentityOptions<Document> {
+  verify: ImpIdentityVerifier<Document>['verify'];
+  trust: ImpIdentityTrust<Document>;
+  now: () => number;
+}
+
+// The broker removes any `Imp-Identity` header a guest sends and adds its own: a compact JWS. A
+// header of any other shape, two joined headers included, never reaches the verifier.
+const compactToken = /^[\w-]+\.[\w-]+\.[\w-]+$/u;
+
+// Proves the sender from the signed identity token of a worker's machine. The issuer and the
+// subject come only from the verified token, and every failure, a throw included, proves nothing.
+// It never logs the token.
+export function createImpIdentitySenderProof<Document>(
+  options: ImpIdentityOptions<Document>,
+): SenderProof {
+  return {
+    prove: async (requirement, headers) => {
+      const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
+      const audience = options.trust.audiences.get(requirement.audienceEnv);
+      const header = headers.get('imp-identity');
+
+      if (
+        documents.length === 0 ||
+        audience === undefined ||
+        header === null ||
+        !compactToken.test(header)
+      ) {
+        return null;
+      }
+
+      try {
+        const verified = await options.verify(header, {
+          issuers: documents,
+          audience,
+          nowSeconds: Math.floor(options.now() / 1000),
+        });
+
+        return verified === null ? null : toSender(verified);
+      } catch {
+        console.error('the identity verifier failed, so the delivery proves no sender');
+
+        return null;
+      }
+    },
+  };
+}
+
+function toSender(verified: VerifiedImp): Sender | null {
+  if (verified.issuer === '' || verified.id === '') {
+    return null;
+  }
+
+  return { issuer: verified.issuer, subject: verified.id, name: verified.name };
+}

package/src/sender-proof/imp-identity/read-imp-identity-trust.ts

@@ -0,0 +1,87 @@
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Result, SenderRequirement, SourceDefinition } from '@pepys/kit';
+import type { ImpIdentityTrust, ImpIdentityVerifier } from './types.ts';
+
+interface TrustReadOptions<Document> {
+  sources: readonly SourceDefinition[];
+
+  // the environment of the ingest process; never a request
+  environment: Readonly<Record<string, string | undefined>>;
+  loadIssuers: ImpIdentityVerifier<Document>['loadIssuers'];
+}
+
+// An unset or empty variable, or an empty folder, leaves its paths with no entry, so they refuse
+// every delivery. A folder that cannot be read, or a document the verifier refuses, fails the
+// start.
+export async function readImpIdentityTrust<Document>(
+  options: TrustReadOptions<Document>,
+): Promise<Result<ImpIdentityTrust<Document>, string>> {
+  const requirements = collectSenderRequirements(options.sources);
+
+  const issuers = new Map<string, readonly Document[]>();
+  const audiences = new Map<string, string>();
+
+  for (const name of new Set(requirements.map((requirement) => requirement.audienceEnv))) {
+    const audience = options.environment[name] ?? '';
+
+    if (audience !== '') {
+      audiences.set(name, audience);
+    }
+  }
+
+  for (const name of new Set(requirements.map((requirement) => requirement.issuersEnv))) {
+    const folder = options.environment[name] ?? '';
+
+    if (folder === '') {
+      continue;
+    }
+
+    const documents = await readFolderDocuments(folder);
+
+    if (!documents.ok) {
+      return { ok: false, failure: `${name}: ${documents.failure}` };
+    }
+
+    if (documents.value.length === 0) {
+      continue;
+    }
+
+    try {
+      issuers.set(name, options.loadIssuers(documents.value));
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : 'the verifier gave no reason';
+
+      return { ok: false, failure: `${name}: the trust documents are invalid: ${reason}` };
+    }
+  }
+
+  return { ok: true, value: { issuers, audiences } };
+}
+
+function collectSenderRequirements(sources: readonly SourceDefinition[]): SenderRequirement[] {
+  return sources.flatMap((source) =>
+    source.live?.kind === 'receive' && source.live.sender !== undefined ? [source.live.sender] : [],
+  );
+}
+
+async function readFolderDocuments(folder: string): Promise<Result<unknown[], string>> {
+  try {
+    const entries = await readdir(folder);
+
+    const files = entries.filter((file) => file.endsWith('.json')).toSorted();
+    const documents: unknown[] = [];
+
+    for (const file of files) {
+      const document: unknown = await Bun.file(join(folder, file)).json();
+
+      documents.push(document);
+    }
+
+    return { ok: true, value: documents };
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : 'no reason given';
+
+    return { ok: false, failure: `the trust documents cannot be read: ${reason}` };
+  }
+}

package/src/sender-proof/imp-identity/types.ts

@@ -0,0 +1,38 @@
+// The imp that a verified identity token names. The shape of the verifier's answer in the
+// `@impz/client/identity` entry of impz.
+export interface VerifiedImp {
+  // the token's `iss`: the impd that signed it
+  readonly issuer: string;
+
+  // the token's `sub`: the imp's ID, unique on one impd only
+  readonly id: string;
+
+  // the imp's name, for logs; a name is free again after a destroy, so it is no key
+  readonly name: string;
+}
+
+// What the deployment trusts, read from its environment at start: the trust documents of each
+// `issuersEnv` and the audience of each `audienceEnv` that its paths name. A name with no entry
+// refuses every delivery on its paths.
+export interface ImpIdentityTrust<Document> {
+  readonly issuers: ReadonlyMap<string, readonly Document[]>;
+  readonly audiences: ReadonlyMap<string, string>;
+}
+
+export interface VerifyOptions<Document> {
+  readonly issuers: readonly Document[];
+  readonly audience: string;
+  readonly nowSeconds: number;
+}
+
+// The verifier of `@impz/client/identity`, which owns the token format and the trust document
+// format. `Document` stays opaque to ingest.
+export interface ImpIdentityVerifier<Document> {
+  // Answers null for a token to refuse: a bad signature, an issuer with no trust document, another
+  // audience, or a token out of its time. Throws on an empty or doubled list of documents.
+  verify: (header: string | null, options: VerifyOptions<Document>) => Promise<VerifiedImp | null>;
+
+  // Parses the documents a deployment pinned. Throws on an invalid document, on none, and on two
+  // with one issuer.
+  loadIssuers: (documents: readonly unknown[]) => Document[];
+}

package/src/sender-proof/none/no-sender-proof.ts

@@ -0,0 +1,7 @@
+import type { SenderProof } from '@pepys/kit';
+
+// Proves no sender, so a path that needs one refuses every delivery. Ingest uses it for a path whose
+// trust documents are missing.
+export const noSenderProof: SenderProof = {
+  prove: () => Promise.resolve(null),
+};

package/src/serve/serve-ingest.ts

@@ -0,0 +1,85 @@
+import { checkDeployment, listReceiveRoutes } from '@pepys/kit';
+import type { Deployment, SourceDefinition, StreamName } from '@pepys/kit';
+import { makeShutdownHandler } from '@pepys/schema';
+import { toTagged } from '@zgeoff/std';
+import { readIngestConfig } from '../config/read-ingest-config.ts';
+import { createS2FactLog } from '../fact-log/s2/create-s2-fact-log.ts';
+import { createIngest } from '../receive/create-ingest.ts';
+import { noSenderProof } from '../sender-proof/none/no-sender-proof.ts';
+
+interface ServeIngestOptions {
+  deployment: Deployment;
+  environment: Readonly<Record<string, string | undefined>>;
+}
+
+// The ingest process of a deployment. It exits 1 at start on a failed deployment check or on any
+// missing variable, so no route ever runs with an empty secret.
+export async function serveIngest(options: ServeIngestOptions): Promise<void> {
+  const checked = checkDeployment(options.deployment);
+
+  if (!checked.ok) {
+    console.error('the deployment fails its check', JSON.stringify(checked.failure));
+    process.exit(1);
+  }
+
+  const sources = checked.value.sources;
+  const config = readIngestConfig(options.environment, collectSecretNames(sources));
+
+  if (!config.ok) {
+    console.error(config.failure.join('\n'));
+    process.exit(1);
+  }
+
+  // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
+  const log = await createS2FactLog({ ...config.value.s2, streams: collectStreams(sources) });
+
+  // No adapter verifies a sender yet, so a path that needs one refuses every delivery.
+  const ingest = createIngest({
+    deployment: checked.value,
+    secrets: config.value.secrets,
+    log,
+    senderProof: noSenderProof,
+    appendRetry: { maxAttempts: 5, retryDelayMs: 200 },
+    now: () => Date.now(),
+  });
+
+  for (const route of listReceiveRoutes(options.deployment)) {
+    console.log(`route ${route.route}`, JSON.stringify(route));
+  }
+
+  const server = Bun.serve({
+    hostname: config.value.host,
+    port: config.value.port,
+    fetch: ingest.handleRequest,
+  });
+
+  console.log(`ingest listening on ${server.hostname}:${server.port}`);
+
+  const shutdown = makeShutdownHandler({
+    stop: () => server.stop(),
+    logError: (message, error) => {
+      console.error(message, error);
+    },
+    exit: (code) => process.exit(code),
+  });
+
+  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.on(signal, () => {
+      void shutdown();
+    });
+  }
+}
+
+function collectSecretNames(sources: readonly SourceDefinition[]): string[] {
+  const names = sources.flatMap((source) =>
+    source.live?.kind === 'receive' ? [source.live.check.secretEnv] : [],
+  );
+
+  return [...new Set(names)];
+}
+
+function collectStreams(sources: readonly SourceDefinition[]): StreamName[] {
+  const streams = sources.flatMap((source) => source.families.map((family) => family.stream));
+
+  return [...new Set(streams)].map((stream) => toTagged<StreamName>(stream));
+}

package/src/source-port/types.ts

@@ -0,0 +1,64 @@
+import type {
+  BlobHash,
+  ConsumerDeclaration,
+  EntityKind,
+  FactEnvelope,
+  LinkTypeDeclaration,
+  Result,
+  StreamName,
+} from '@pepys/kit';
+
+export interface FactFamily<Payload> {
+  stream: StreamName;
+  types: readonly string[];
+  api: string;
+  parsePayload: (raw: unknown) => Result<Payload, { reason: string }>;
+}
+
+// Provisional: source-retention
+export interface SourceDefinition {
+  name: string;
+  families: readonly FactFamily<unknown>[];
+  entityKinds: readonly EntityKind[];
+  linkTypes: readonly LinkTypeDeclaration[];
+  consumers: readonly ConsumerDeclaration[];
+  live: ReceivePath | PullPath;
+  recovery: PullPath;
+  backfill: PullPath;
+  resync: PullPath;
+}
+
+export interface ReceivePath {
+  kind: 'receive';
+  route: string;
+  receive: (request: Request) => Promise<Received>;
+}
+
+export type Received =
+  | { kind: 'refused'; status: 400 | 401 | 405 | 422; reason: string }
+  | { kind: 'ignored' }
+  | { kind: 'batch'; batch: Batch };
+
+export interface PullPath {
+  kind: 'pull';
+  name: string;
+  intervalMs: number | null;
+  pull: (cursor: string | null) => Promise<Pulled>;
+}
+
+// Provisional: source-retention
+export type Pulled =
+  | { kind: 'batch'; batch: Batch; cursor: string; isDone: boolean }
+  | { kind: 'expired' }
+  | { kind: 'unavailable'; retryAfterMs: number }
+  | { kind: 'rejected'; reason: string };
+
+export interface Batch {
+  facts: readonly PendingFact[];
+}
+
+export interface PendingFact {
+  stream: StreamName;
+  fact: FactEnvelope<unknown>;
+  blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
+}