@pepys/ingest 0.0.0-stage → 0.1.0

package/src/receive/resync-marks/create-resync-marks.ts

@@ -0,0 +1,111 @@
+import type { EntityID, ResyncMark } from '@pepys/kit';
+import { collectReturnedEntities } from './collect-returned-entities.ts';
+import { collectTiedEntities } from './collect-tied-entities.ts';
+import type { LastBody, StoredFactTime } from './collect-tied-entities.ts';
+
+// The entities whose row may be behind the source, which a source's resync pull reads again. The
+// pull reads the marks, and removes each one only after its fact is stored, so a failed pull keeps
+// it. In memory only, so a restart loses them.
+export interface ResyncMarks {
+  list: (source: string) => readonly ResyncMark[];
+  remove: (source: string, entity: EntityID) => void;
+}
+
+// What the receive route of one source tells the marks.
+export interface SourceMarks {
+  noteStored: (digest: string, facts: readonly StoredFactTime[]) => void;
+  noteCopy: (digest: string, entities: readonly EntityID[]) => void;
+}
+
+interface SourceState {
+  lastBodies: Map<EntityID, LastBody>;
+
+  // each marked entity, with the locator of the last fact stored for it
+  marks: Map<EntityID, string | undefined>;
+}
+
+export interface ResyncMarkStore {
+  marks: ResyncMarks;
+  createSourceMarks: (source: string) => SourceMarks;
+}
+
+// `capacity` bounds the marks and the last bodies of each source; the oldest go first
+interface ResyncMarksOptions {
+  capacity: number;
+}
+
+export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore {
+  const sources = new Map<string, SourceState>();
+
+  const getState = (source: string): SourceState => {
+    const known = sources.get(source);
+
+    if (known !== undefined) {
+      return known;
+    }
+
+    const state: SourceState = { lastBodies: new Map(), marks: new Map() };
+
+    sources.set(source, state);
+
+    return state;
+  };
+
+  return {
+    marks: {
+      list: (source) =>
+        [...(sources.get(source)?.marks ?? [])].map(([entity, locator]) => ({ entity, locator })),
+      remove: (source, entity) => {
+        sources.get(source)?.marks.delete(entity);
+      },
+    },
+    createSourceMarks: (source) => {
+      const state = getState(source);
+
+      return {
+        noteStored: (digest, facts) => {
+          const tied = collectTiedEntities(state.lastBodies, digest, facts);
+
+          for (const fact of facts) {
+            state.lastBodies.delete(fact.entity);
+            state.lastBodies.set(fact.entity, { digest, at: fact.at, locator: fact.locator });
+          }
+
+          addMarks(state, tied, options);
+          trimToCapacity(state.lastBodies, options.capacity);
+        },
+        noteCopy: (digest, entities) => {
+          addMarks(state, collectReturnedEntities(state.lastBodies, digest, entities), options);
+        },
+      };
+    },
+  };
+}
+
+// The locator comes from the last body stored for the entity, which the caller has noted first.
+function addMarks(
+  state: SourceState,
+  entities: readonly EntityID[],
+  options: ResyncMarksOptions,
+): void {
+  for (const entity of entities) {
+    state.marks.delete(entity);
+    state.marks.set(entity, state.lastBodies.get(entity)?.locator);
+  }
+
+  trimToCapacity(state.marks, options.capacity);
+}
+
+// drops the oldest entries first: a Map and a Set keep insertion order
+function trimToCapacity(
+  entries: Map<EntityID, LastBody> | Map<EntityID, string | undefined>,
+  capacity: number,
+): void {
+  for (const key of entries.keys()) {
+    if (entries.size <= capacity) {
+      return;
+    }
+
+    entries.delete(key);
+  }
+}

package/src/receive/write-batch.ts

@@ -0,0 +1,79 @@
+import type { StreamName } from '@pepys/kit';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import type { BatchPlan } from './plan-batch.ts';
+
+export type BatchOutcome = 'stored' | 'refused' | 'not-stored';
+
+// One chain of turns per stream, for the whole process. A delivery holds a stream's turn for all of
+// its appends on that stream, so the appends of two deliveries never interleave.
+export type StreamTurns = Map<StreamName, Promise<void>>;
+
+interface WriteBatchDependencies {
+  append: BatchAppender;
+  turns: StreamTurns;
+}
+
+// The streams append at the same time, and the appends of one stream one after the other. Every
+// stream settles before the answer, so a failure on one stream never leaves another in flight.
+export async function writeBatch(
+  plan: BatchPlan,
+  dependencies: WriteBatchDependencies,
+): Promise<BatchOutcome> {
+  const settled = await Promise.all(
+    [...plan].map(([stream, runs]) =>
+      withStreamTurn(dependencies.turns, stream, async () => {
+        for (const run of runs) {
+          const appended = await dependencies.append(stream, run);
+
+          if (!appended.ok) {
+            console.error(`an append to ${stream} failed`, appended.failure);
+
+            return appended.failure.kind;
+          }
+        }
+
+        return 'stored' as const;
+      }),
+    ),
+  );
+
+  return pickBatchOutcome(settled);
+}
+
+// A throw is a refusal the log gives no outcome for, such as a bad access token.
+async function withStreamTurn(
+  turns: StreamTurns,
+  stream: StreamName,
+  work: () => Promise<BatchOutcome>,
+): Promise<BatchOutcome> {
+  const previous = turns.get(stream) ?? Promise.resolve();
+  const done = Promise.withResolvers<void>();
+
+  turns.set(stream, done.promise);
+
+  try {
+    await previous;
+
+    return await work();
+  } catch (error: unknown) {
+    console.error(`an append to ${stream} threw`, error);
+
+    return 'refused';
+  } finally {
+    done.resolve();
+
+    if (turns.get(stream) === done.promise) {
+      turns.delete(stream);
+    }
+  }
+}
+
+// A refusal wins over `not-stored`: a repeat cannot mend it, so the sender must not be told to
+// repeat.
+function pickBatchOutcome(settled: readonly BatchOutcome[]): BatchOutcome {
+  if (settled.includes('refused')) {
+    return 'refused';
+  }
+
+  return settled.includes('not-stored') ? 'not-stored' : 'stored';
+}

package/src/sender-proof/fake/create-fake-sender-proof.ts

@@ -0,0 +1,14 @@
+import type { Sender, SenderProof } from '@pepys/kit';
+
+// The test stand-in for a verifier of signed identity tokens: the value of the `imp-identity`
+// header proves the sender it is mapped to, and any other value proves nothing.
+export function createFakeSenderProof(senders: Readonly<Record<string, Sender>>): SenderProof {
+  return {
+    prove: (_requirement, headers) => {
+      const token = headers.get('imp-identity');
+      const sender = token !== null && Object.hasOwn(senders, token) ? senders[token] : undefined;
+
+      return Promise.resolve(sender ?? null);
+    },
+  };
+}

package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts

@@ -0,0 +1,42 @@
+import type { verifyImpIdentity } from '@impz/client/identity';
+import type { SenderProof } from '@pepys/kit';
+import type { ImpIdentityTrust } from './types.ts';
+
+interface ImpIdentityOptions {
+  verify: typeof verifyImpIdentity;
+  trust: ImpIdentityTrust;
+  now: () => number;
+}
+
+// Proves the sender from the signed identity token in the `imp-identity` header. The issuer and
+// the subject come only from the verified token, and every failure, a throw included, proves
+// nothing. It never logs the token.
+export function createImpIdentitySenderProof(options: ImpIdentityOptions): SenderProof {
+  return {
+    prove: async (requirement, headers) => {
+      const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
+      const audience = options.trust.audiences.get(requirement.audienceEnv);
+
+      // The verifier throws on an empty list of documents, so a path with none refuses here.
+      if (documents.length === 0 || audience === undefined) {
+        return null;
+      }
+
+      try {
+        const verified = await options.verify(headers.get('imp-identity'), {
+          issuers: documents,
+          audience,
+          nowSeconds: Math.floor(options.now() / 1000),
+        });
+
+        return verified === null
+          ? null
+          : { issuer: verified.issuer, subject: verified.id, name: verified.name };
+      } catch {
+        console.error('the identity verifier failed, so the delivery proves no sender');
+
+        return null;
+      }
+    },
+  };
+}

package/src/sender-proof/imp-identity/read-imp-identity-sender-proof.ts

@@ -0,0 +1,35 @@
+import { verifyImpIdentity } from '@impz/client/identity';
+import type { Result, SenderProof, SourceDefinition } from '@pepys/kit';
+import { createImpIdentitySenderProof } from './create-imp-identity-sender-proof.ts';
+import { readImpIdentityTrust } from './read-imp-identity-trust.ts';
+
+interface SenderProofReadOptions {
+  sources: readonly SourceDefinition[];
+  environment: Readonly<Record<string, string | undefined>>;
+  now: () => number;
+}
+
+// The sender proof that `serveIngest` runs: the trust of the deployment's environment and the
+// verifier of `@impz/client/identity`. It fails only when a set folder cannot be read or holds an
+// invalid document.
+export async function readImpIdentitySenderProof(
+  options: SenderProofReadOptions,
+): Promise<Result<SenderProof, string>> {
+  const trust = await readImpIdentityTrust({
+    sources: options.sources,
+    environment: options.environment,
+  });
+
+  if (!trust.ok) {
+    return trust;
+  }
+
+  return {
+    ok: true,
+    value: createImpIdentitySenderProof({
+      verify: verifyImpIdentity,
+      trust: trust.value,
+      now: options.now,
+    }),
+  };
+}

package/src/sender-proof/imp-identity/read-imp-identity-trust.ts

@@ -0,0 +1,88 @@
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { loadIdentityIssuers } from '@impz/client/identity';
+import type { IdentityTrustDocument } from '@impz/client/identity';
+import type { Result, SenderRequirement, SourceDefinition } from '@pepys/kit';
+import type { ImpIdentityTrust } from './types.ts';
+
+interface TrustReadOptions {
+  sources: readonly SourceDefinition[];
+
+  // the environment of the ingest process; never a request
+  environment: Readonly<Record<string, string | undefined>>;
+}
+
+// An unset or empty variable, or an empty folder, leaves its paths with no entry, so they refuse
+// every delivery. A folder that cannot be read, or a document that the loader refuses, fails the
+// start.
+export async function readImpIdentityTrust(
+  options: TrustReadOptions,
+): Promise<Result<ImpIdentityTrust, string>> {
+  const requirements = collectSenderRequirements(options.sources);
+
+  const issuers = new Map<string, readonly IdentityTrustDocument[]>();
+  const audiences = new Map<string, string>();
+
+  for (const name of new Set(requirements.map((requirement) => requirement.audienceEnv))) {
+    const audience = options.environment[name] ?? '';
+
+    if (audience !== '') {
+      audiences.set(name, audience);
+    }
+  }
+
+  for (const name of new Set(requirements.map((requirement) => requirement.issuersEnv))) {
+    const folder = options.environment[name] ?? '';
+
+    if (folder === '') {
+      continue;
+    }
+
+    const documents = await readFolderDocuments(folder);
+
+    if (!documents.ok) {
+      return { ok: false, failure: `${name}: ${documents.failure}` };
+    }
+
+    if (documents.value.length === 0) {
+      continue;
+    }
+
+    try {
+      issuers.set(name, loadIdentityIssuers(documents.value));
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : 'the loader gave no reason';
+
+      return { ok: false, failure: `${name}: the trust documents are invalid: ${reason}` };
+    }
+  }
+
+  return { ok: true, value: { issuers, audiences } };
+}
+
+function collectSenderRequirements(sources: readonly SourceDefinition[]): SenderRequirement[] {
+  return sources.flatMap((source) =>
+    source.live?.kind === 'receive' && source.live.sender !== undefined ? [source.live.sender] : [],
+  );
+}
+
+async function readFolderDocuments(folder: string): Promise<Result<unknown[], string>> {
+  try {
+    const entries = await readdir(folder);
+
+    const files = entries.filter((file) => file.endsWith('.json')).toSorted();
+    const documents: unknown[] = [];
+
+    for (const file of files) {
+      const document: unknown = await Bun.file(join(folder, file)).json();
+
+      documents.push(document);
+    }
+
+    return { ok: true, value: documents };
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : 'no reason given';
+
+    return { ok: false, failure: `the trust documents cannot be read: ${reason}` };
+  }
+}

package/src/sender-proof/imp-identity/types.ts

@@ -0,0 +1,9 @@
+import type { IdentityTrustDocument } from '@impz/client/identity';
+
+// What the deployment trusts, read from its environment at start: the trust documents of each
+// `issuersEnv` and the audience of each `audienceEnv` that its paths name. A name with no entry
+// refuses every delivery on its paths.
+export interface ImpIdentityTrust {
+  readonly issuers: ReadonlyMap<string, readonly IdentityTrustDocument[]>;
+  readonly audiences: ReadonlyMap<string, string>;
+}

package/src/sender-proof/none/no-sender-proof.ts

@@ -0,0 +1,7 @@
+import type { SenderProof } from '@pepys/kit';
+
+// Proves no sender, so a path that needs one refuses every delivery. Ingest uses it for a path whose
+// trust documents are missing.
+export const noSenderProof: SenderProof = {
+  prove: () => Promise.resolve(null),
+};

package/src/serve/serve-ingest.ts

@@ -0,0 +1,107 @@
+import { checkDeployment, listReceiveRoutes } from '@pepys/kit';
+import type { Deployment, SourceDefinition, StreamName } from '@pepys/kit';
+import { makeShutdownHandler } from '@pepys/schema';
+import { toTagged } from '@zgeoff/std';
+import { readIngestConfig } from '../config/read-ingest-config.ts';
+import { createS2FactLog } from '../fact-log/s2/create-s2-fact-log.ts';
+import { createIngest } from '../receive/create-ingest.ts';
+import { readImpIdentitySenderProof } from '../sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
+import { startPullTimers } from './start-pull-timers.ts';
+
+interface ServeIngestOptions {
+  deployment: Deployment;
+  environment: Readonly<Record<string, string | undefined>>;
+}
+
+// The ingest process of a deployment. It exits 1 at start on a failed deployment check, on any
+// missing variable, or on a trust folder it cannot load, so no route ever runs with an empty secret.
+// A path whose trust documents or audience are unset starts, and refuses every delivery.
+export async function serveIngest(options: ServeIngestOptions): Promise<void> {
+  const checked = checkDeployment(options.deployment);
+
+  if (!checked.ok) {
+    console.error('the deployment fails its check', JSON.stringify(checked.failure));
+    process.exit(1);
+  }
+
+  const sources = checked.value.sources;
+  const config = readIngestConfig(options.environment, collectSecretNames(sources));
+
+  if (!config.ok) {
+    console.error(config.failure.join('\n'));
+    process.exit(1);
+  }
+
+  const senderProof = await readImpIdentitySenderProof({
+    sources,
+    environment: options.environment,
+    now: () => Date.now(),
+  });
+
+  if (!senderProof.ok) {
+    console.error(senderProof.failure);
+    process.exit(1);
+  }
+
+  // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
+  const log = await createS2FactLog({ ...config.value.s2, streams: collectStreams(sources) });
+
+  const ingest = createIngest({
+    deployment: checked.value,
+    secrets: config.value.secrets,
+    log,
+    senderProof: senderProof.value,
+    appendRetry: { maxAttempts: 5, retryDelayMs: 200 },
+    now: () => Date.now(),
+  });
+
+  for (const route of listReceiveRoutes(options.deployment)) {
+    console.log(`route ${route.route}`, JSON.stringify(route));
+  }
+
+  const stopPulls = startPullTimers(sources, ingest.runPull);
+
+  const server = Bun.serve({
+    hostname: config.value.host,
+    port: config.value.port,
+    fetch: ingest.handleRequest,
+  });
+
+  console.log(`ingest listening on ${server.hostname}:${server.port}`);
+
+  const shutdown = makeShutdownHandler({
+    stop: () => {
+      stopPulls();
+
+      return server.stop();
+    },
+    logError: (message, error) => {
+      console.error(message, error);
+    },
+    exit: (code) => process.exit(code),
+  });
+
+  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+    process.on(signal, () => {
+      void shutdown();
+    });
+  }
+}
+
+// every receive path's secret, and every name a pull path's `env` lists
+function collectSecretNames(sources: readonly SourceDefinition[]): string[] {
+  const names = sources.flatMap((source) => [
+    ...(source.live?.kind === 'receive' ? [source.live.check.secretEnv] : []),
+    ...[source.live, source.recovery, source.backfill, source.resync].flatMap((path) =>
+      path?.kind === 'pull' ? (path.env ?? []) : [],
+    ),
+  ]);
+
+  return [...new Set(names)];
+}
+
+function collectStreams(sources: readonly SourceDefinition[]): StreamName[] {
+  const streams = sources.flatMap((source) => source.families.map((family) => family.stream));
+
+  return [...new Set(streams)].map((stream) => toTagged<StreamName>(stream));
+}

package/src/serve/start-pull-timers.ts

@@ -0,0 +1,69 @@
+import type { SourceDefinition } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { PullPathName, PullRun, PullRunner } from '../pull/make-pull-runner.ts';
+
+const pullPathNames: readonly PullPathName[] = ['live', 'recovery', 'backfill', 'resync'];
+
+// One timer per pull path with an interval. The cursor lives in memory, so a restart starts each
+// pull from no cursor. A run never overlaps the one before it. Returns the function that stops them.
+export function startPullTimers(
+  sources: readonly SourceDefinition[],
+  runPull: PullRunner,
+): () => void {
+  const timers = sources.flatMap((source) =>
+    pullPathNames.flatMap((pathName) => {
+      const path = source[pathName];
+
+      if (path?.kind !== 'pull' || path.intervalMs === null) {
+        return [];
+      }
+
+      let cursor: string | null = null;
+      let isRunning = false;
+
+      const runOnce = async (): Promise<void> => {
+        if (isRunning) {
+          return;
+        }
+
+        isRunning = true;
+
+        try {
+          const run = await runPull(source.name, pathName, cursor);
+
+          cursor = pickNextCursor(run, cursor);
+
+          if (run.kind !== 'stored') {
+            console.error(`the ${pathName} pull of ${source.name} did not store`, run);
+          }
+        } catch (error: unknown) {
+          console.error(`the ${pathName} pull of ${source.name} threw`, error);
+        } finally {
+          isRunning = false;
+        }
+      };
+
+      return [
+        setInterval(() => {
+          void runOnce();
+        }, path.intervalMs),
+      ];
+    }),
+  );
+
+  return () => {
+    for (const timer of timers) {
+      clearInterval(timer);
+    }
+  };
+}
+
+// An expired cursor starts the pull again from none; any other failure keeps the cursor.
+function pickNextCursor(run: PullRun, cursor: string | null): string | null {
+  return match(run)
+    .with({ kind: 'stored' }, (stored) => stored.cursor)
+    .with({ kind: 'expired' }, () => null)
+    .with({ kind: 'not-stored' }, { kind: 'refused' }, { kind: 'unavailable' }, () => cursor)
+    .with({ kind: 'rejected' }, () => cursor)
+    .exhaustive();
+}

package/src/source-port/types.ts

@@ -0,0 +1,64 @@
+import type {
+  BlobHash,
+  ConsumerDeclaration,
+  EntityKind,
+  FactEnvelope,
+  LinkTypeDeclaration,
+  Result,
+  StreamName,
+} from '@pepys/kit';
+
+export interface FactFamily<Payload> {
+  stream: StreamName;
+  types: readonly string[];
+  api: string;
+  parsePayload: (raw: unknown) => Result<Payload, { reason: string }>;
+}
+
+// Provisional: source-retention
+export interface SourceDefinition {
+  name: string;
+  families: readonly FactFamily<unknown>[];
+  entityKinds: readonly EntityKind[];
+  linkTypes: readonly LinkTypeDeclaration[];
+  consumers: readonly ConsumerDeclaration[];
+  live: ReceivePath | PullPath;
+  recovery: PullPath;
+  backfill: PullPath;
+  resync: PullPath;
+}
+
+export interface ReceivePath {
+  kind: 'receive';
+  route: string;
+  receive: (request: Request) => Promise<Received>;
+}
+
+export type Received =
+  | { kind: 'refused'; status: 400 | 401 | 405 | 422; reason: string }
+  | { kind: 'ignored' }
+  | { kind: 'batch'; batch: Batch };
+
+export interface PullPath {
+  kind: 'pull';
+  name: string;
+  intervalMs: number | null;
+  pull: (cursor: string | null) => Promise<Pulled>;
+}
+
+// Provisional: source-retention
+export type Pulled =
+  | { kind: 'batch'; batch: Batch; cursor: string; isDone: boolean }
+  | { kind: 'expired' }
+  | { kind: 'unavailable'; retryAfterMs: number }
+  | { kind: 'rejected'; reason: string };
+
+export interface Batch {
+  facts: readonly PendingFact[];
+}
+
+export interface PendingFact {
+  stream: StreamName;
+  fact: FactEnvelope<unknown>;
+  blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
+}