@pepys/ingest 0.0.0-stage → 0.1.0

package/src/fact-log/memory/create-memory-fact-log.ts

@@ -0,0 +1,174 @@
+import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+import { toTagged } from '@zgeoff/std';
+import invariant from 'tiny-invariant';
+import { checkAppendBatch } from '../check-append-batch.ts';
+import type { AppendBatchOutcome, FactLog, StoredFact } from '../fact-log.ts';
+import { parseStoredFact } from '../parse-stored-fact.ts';
+
+interface OtherWriter {
+  thenAppend: readonly FactEnvelope<unknown>[];
+}
+
+export interface MemoryFactLog extends FactLog {
+  // The next append is stored whole, and its outcome is reported as unknown.
+  loseNextReply: () => void;
+
+  // The next append is not stored and its outcome is reported as unknown. Then `thenAppend` is
+  // stored at the tail, as a second writer on the stream would store it.
+  dropNextAppend: (other: OtherWriter) => void;
+
+  // The next append reports unknown and is not stored yet. It is stored, whole, just before the
+  // append after it is checked, so that one gets a tail mismatch.
+  landNextAppendLate: () => void;
+}
+
+interface MemoryRecord {
+  body: string;
+  appendedAtMs: number;
+}
+
+type Fault =
+  | { kind: 'lose-reply' }
+  | { kind: 'drop'; thenAppend: readonly FactEnvelope<unknown>[] }
+  | { kind: 'land-late' };
+
+interface LateAppend {
+  stream: StreamName;
+  bodies: readonly string[];
+}
+
+interface MemoryState {
+  records: Map<StreamName, MemoryRecord[]>;
+  fault: Fault | null;
+  late: LateAppend | null;
+}
+
+export function createMemoryFactLog(streams: readonly StreamName[]): MemoryFactLog {
+  const state: MemoryState = {
+    records: new Map(streams.map((stream) => [stream, []])),
+    fault: null,
+    late: null,
+  };
+
+  return {
+    appendBatch: (stream, facts, expectedTail) =>
+      Promise.resolve(appendBatch(state, stream, facts, expectedTail)),
+    read: (stream, from, count) =>
+      Promise.resolve(collectStoredFacts(state.records.get(stream), from, count)),
+    readTail: (stream) => Promise.resolve(getTail(state.records.get(stream))),
+    loseNextReply: () => {
+      state.fault = { kind: 'lose-reply' };
+    },
+    dropNextAppend: (other) => {
+      state.fault = { kind: 'drop', thenAppend: other.thenAppend };
+    },
+    landNextAppendLate: () => {
+      state.fault = { kind: 'land-late' };
+    },
+  };
+}
+
+function appendBatch(
+  state: MemoryState,
+  stream: StreamName,
+  facts: readonly FactEnvelope<unknown>[],
+  expectedTail: StreamOffset,
+): AppendBatchOutcome {
+  landLateAppend(state);
+
+  const fault = state.fault;
+
+  state.fault = null;
+
+  const checked = checkAppendBatch(facts);
+
+  if (!checked.ok) {
+    return { kind: 'rejected', ...checked.failure };
+  }
+
+  const stored = state.records.get(stream);
+
+  if (stored === undefined) {
+    return { kind: 'rejected', reason: 'unknown-stream' };
+  }
+
+  if (expectedTail !== stored.length) {
+    return { kind: 'rejected', reason: 'tail-mismatch' };
+  }
+
+  if (fault?.kind === 'drop') {
+    pushOtherWriter(stored, fault.thenAppend);
+
+    return { kind: 'unknown' };
+  }
+
+  if (fault?.kind === 'land-late') {
+    state.late = { stream, bodies: checked.value };
+
+    return { kind: 'unknown' };
+  }
+
+  pushBodies(stored, checked.value);
+
+  return fault === null
+    ? { kind: 'acknowledged', firstOffset: expectedTail, count: checked.value.length }
+    : { kind: 'unknown' };
+}
+
+// The append stays valid until it lands: nothing is stored between its refusal of a reply and the
+// next call, so the tail is still the one it aimed at.
+function landLateAppend(state: MemoryState): void {
+  const late = state.late;
+
+  state.late = null;
+
+  const stored = late === null ? undefined : state.records.get(late.stream);
+
+  if (late !== null && stored !== undefined) {
+    pushBodies(stored, late.bodies);
+  }
+}
+
+function pushBodies(stored: MemoryRecord[], bodies: readonly string[]): void {
+  const appendedAtMs = Date.now();
+
+  for (const body of bodies) {
+    stored.push({ body, appendedAtMs });
+  }
+}
+
+function pushOtherWriter(stored: MemoryRecord[], facts: readonly FactEnvelope<unknown>[]): void {
+  const checked = checkAppendBatch(facts);
+
+  invariant(checked.ok, 'the facts of a second writer pass the append checks');
+  pushBodies(stored, checked.value);
+}
+
+type StreamRead<Value> = Result<Value, { reason: 'unknown-stream' }>;
+
+function collectStoredFacts(
+  stored: readonly MemoryRecord[] | undefined,
+  from: StreamOffset,
+  count: number,
+): StreamRead<readonly StoredFact[]> {
+  if (stored === undefined) {
+    return { ok: false, failure: { reason: 'unknown-stream' } };
+  }
+
+  return {
+    ok: true,
+    value: stored
+      .slice(from, from + count)
+      .map((record, index) =>
+        parseStoredFact(toTagged<StreamOffset>(from + index), record.body, record.appendedAtMs),
+      ),
+  };
+}
+
+function getTail(stored: readonly MemoryRecord[] | undefined): StreamRead<StreamOffset> {
+  if (stored === undefined) {
+    return { ok: false, failure: { reason: 'unknown-stream' } };
+  }
+
+  return { ok: true, value: toTagged<StreamOffset>(stored.length) };
+}

package/src/fact-log/parse-stored-body.ts

@@ -0,0 +1,18 @@
+import { parseFactEnvelope } from '@pepys/kit';
+import type { FactEnvelope } from '@pepys/kit';
+
+// Checks the JSON a log would store, so a field that JSON drops, such as a function, cannot leave
+// a stored record that fails the envelope parse on read.
+export function parseStoredBody(fact: FactEnvelope<unknown>): string | null {
+  let body: string;
+
+  try {
+    body = JSON.stringify(fact);
+  } catch {
+    return null;
+  }
+
+  const json: unknown = JSON.parse(body);
+
+  return parseFactEnvelope(json).ok ? body : null;
+}

package/src/fact-log/parse-stored-fact.ts

@@ -0,0 +1,17 @@
+import { parseFactEnvelope } from '@pepys/kit';
+import type { StreamOffset } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import type { StoredFact } from './fact-log.ts';
+
+export function parseStoredFact(
+  offset: StreamOffset,
+  body: string,
+  appendedAtMs: number,
+): StoredFact {
+  const json: unknown = JSON.parse(body);
+  const fact = parseFactEnvelope(json);
+
+  invariant(fact.ok, 'the log holds only envelopes that an append parsed');
+
+  return { offset, fact: fact.value, body, appendedAtMs };
+}

package/src/fact-log/pick-settlement.ts

@@ -0,0 +1,20 @@
+import type { FactID } from '@pepys/kit';
+import type { StoredFact } from './fact-log.ts';
+
+// `landed`: the run is stored whole at the offsets the attempt aimed at. `not-landed`: nothing is
+// there, so the attempt may still be in flight. `taken`: another writer took the offsets, so the
+// attempt can never land.
+export type Settlement = 'landed' | 'not-landed' | 'taken';
+
+// `stored` holds what the log gives for the offsets `[t, t + n)` that an attempt aimed at. One
+// append is whole, so a match of fewer than n IDs is another writer's records, never part of the run.
+export function pickSettlement(ids: readonly FactID[], stored: readonly StoredFact[]): Settlement {
+  if (stored.length === 0) {
+    return 'not-landed';
+  }
+
+  const isSameRun =
+    stored.length === ids.length && stored.every((record, index) => record.fact.id === ids[index]);
+
+  return isSameRun ? 'landed' : 'taken';
+}

package/src/fact-log/plan-appends.ts

@@ -0,0 +1,45 @@
+import type { FactEnvelope, Result } from '@pepys/kit';
+import { appendLimits, toMeteredBytes } from './check-append-batch.ts';
+import { parseStoredBody } from './parse-stored-body.ts';
+
+type Run = readonly FactEnvelope<unknown>[];
+
+// Cuts the facts of one stream, in order, into runs that each stay inside the append limits. A run
+// closes when the next fact would make it 1001 records or more than 1,048,576 metered bytes. Fact
+// `index` is too large when it alone meters more than one append takes.
+export function planAppends(
+  facts: readonly FactEnvelope<unknown>[],
+): Result<readonly Run[], { reason: 'too-large'; index: number }> {
+  const runs: FactEnvelope<unknown>[][] = [];
+  let run: FactEnvelope<unknown>[] = [];
+  let runBytes = 0;
+
+  for (const [index, fact] of facts.entries()) {
+    // A fact that fails the envelope check meters as 0: the append refuses it by its own check.
+    const bytes = toMeteredBytes(parseStoredBody(fact) ?? '');
+
+    if (bytes > appendLimits.meteredBytes) {
+      return { ok: false, failure: { reason: 'too-large', index } };
+    }
+
+    const isFull =
+      run.length + 1 > appendLimits.records || runBytes + bytes > appendLimits.meteredBytes;
+
+    if (isFull) {
+      runs.push(run);
+
+      run = [];
+      runBytes = 0;
+    }
+
+    run.push(fact);
+
+    runBytes += bytes;
+  }
+
+  if (run.length > 0) {
+    runs.push(run);
+  }
+
+  return { ok: true, value: runs };
+}

package/src/fact-log/s2/create-s2-fact-log.ts

@@ -0,0 +1,171 @@
+import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+import {
+  AppendInput,
+  AppendRecord,
+  RangeNotSatisfiableError,
+  S2,
+  S2Error,
+} from '@s2-dev/streamstore';
+import type { S2Basin, StreamConfig } from '@s2-dev/streamstore';
+import { toTagged } from '@zgeoff/std';
+import { checkAppendBatch } from '../check-append-batch.ts';
+import type { AppendBatchOutcome, FactLog, StoredFact } from '../fact-log.ts';
+import { parseStoredFact } from '../parse-stored-fact.ts';
+import { findAppendOutcome, isUnknownStream } from './find-append-outcome.ts';
+
+interface S2FactLogParameters {
+  endpoint: string;
+  accessToken: string;
+  basin: string;
+  streams: readonly StreamName[];
+}
+
+// Creates the basin and the streams when they are missing. A stream that exists keeps its records
+// and has its retention set to infinite.
+export async function createS2FactLog(parameters: S2FactLogParameters): Promise<FactLog> {
+  const options = {
+    accessToken: parameters.accessToken,
+    endpoints: { account: parameters.endpoint, basin: parameters.endpoint },
+    requestTimeoutMillis: 5000,
+  };
+
+  const client = new S2(options);
+
+  // A client retry could store a record twice before the writer knows; the writer owns retries.
+  const appendClient = new S2({ ...options, retry: { maxAttempts: 1 } });
+
+  await ignoreExisting(client.basins.create({ basin: parameters.basin }));
+
+  const basin = client.basin(parameters.basin);
+  const appendBasin = appendClient.basin(parameters.basin);
+
+  for (const stream of parameters.streams) {
+    await createInfiniteStream(basin, stream);
+  }
+
+  return {
+    appendBatch: (stream, facts, expectedTail) =>
+      appendBatch(appendBasin, stream, facts, expectedTail),
+    read: (stream, from, count) => readFacts(basin, stream, from, count),
+    readTail: (stream) => readTail(basin, stream),
+  };
+}
+
+async function ignoreExisting(creation: Promise<unknown>): Promise<boolean> {
+  try {
+    await creation;
+
+    return true;
+  } catch (error: unknown) {
+    if (error instanceof S2Error && error.status === 409) {
+      return false;
+    }
+
+    throw error;
+  }
+}
+
+// log.md: pepys creates every stream with infinite retention, because the default is 7 days.
+const infiniteRetention: StreamConfig = { retentionPolicy: { infinite: {} } };
+
+async function createInfiniteStream(basin: S2Basin, stream: StreamName): Promise<void> {
+  const isCreated = await ignoreExisting(
+    basin.streams.create({ stream, config: infiniteRetention }),
+  );
+
+  if (!isCreated) {
+    await basin.streams.reconfigure({ stream, ...infiniteRetention });
+  }
+}
+
+async function appendBatch(
+  basin: S2Basin,
+  stream: StreamName,
+  facts: readonly FactEnvelope<unknown>[],
+  expectedTail: StreamOffset,
+): Promise<AppendBatchOutcome> {
+  const checked = checkAppendBatch(facts);
+
+  if (!checked.ok) {
+    return { kind: 'rejected', ...checked.failure };
+  }
+
+  // The checks hold every limit, so `create` does not throw.
+  const input = AppendInput.create(
+    checked.value.map((body) => AppendRecord.string({ body })),
+    { matchSeqNum: expectedTail },
+  );
+
+  try {
+    const ack = await basin.stream(stream).append(input);
+
+    return {
+      kind: 'acknowledged',
+      firstOffset: toTagged<StreamOffset>(ack.start.seqNum),
+      count: ack.end.seqNum - ack.start.seqNum,
+    };
+  } catch (error: unknown) {
+    const outcome = findAppendOutcome(error);
+
+    if (outcome === null) {
+      throw error;
+    }
+
+    if (outcome.kind === 'unknown') {
+      console.error('an s2 append has an unknown outcome', error);
+    }
+
+    return outcome;
+  }
+}
+
+type StreamRead<Value> = Result<Value, { reason: 'unknown-stream' }>;
+
+async function readFacts(
+  basin: S2Basin,
+  stream: StreamName,
+  from: StreamOffset,
+  count: number,
+): Promise<StreamRead<readonly StoredFact[]>> {
+  try {
+    const batch = await basin
+      .stream(stream)
+      .read({ start: { from: { seqNum: from } }, stop: { limits: { count } } });
+
+    return {
+      ok: true,
+      value: batch.records.map((record) =>
+        parseStoredFact(
+          toTagged<StreamOffset>(record.seqNum),
+          record.body,
+          record.timestamp.getTime(),
+        ),
+      ),
+    };
+  } catch (error: unknown) {
+    // s2-lite answers 416 to a read that starts at the tail or past it.
+    if (error instanceof RangeNotSatisfiableError) {
+      return { ok: true, value: [] };
+    }
+
+    if (isUnknownStream(error)) {
+      return { ok: false, failure: { reason: 'unknown-stream' } };
+    }
+
+    throw error;
+  }
+}
+
+async function readTail(basin: S2Basin, stream: StreamName): Promise<StreamRead<StreamOffset>> {
+  try {
+    const response = await basin.stream(stream).checkTail();
+
+    return { ok: true, value: toTagged<StreamOffset>(response.tail.seqNum) };
+  } catch (error: unknown) {
+    if (isUnknownStream(error)) {
+      return { ok: false, failure: { reason: 'unknown-stream' } };
+    }
+
+    throw error;
+  }
+}

package/src/fact-log/s2/find-append-outcome.ts

@@ -0,0 +1,35 @@
+import { S2Error, SeqNumMismatchError } from '@s2-dev/streamstore';
+import type { AppendBatchOutcome } from '../fact-log.ts';
+
+// A client error from the server means the record was not stored; 408 and 429 ask for a retry. A
+// refusal the port has no reason for, such as a bad token, returns null for the caller to throw.
+export function findAppendOutcome(error: unknown): AppendBatchOutcome | null {
+  if (error instanceof SeqNumMismatchError) {
+    return { kind: 'rejected', reason: 'tail-mismatch' };
+  }
+
+  if (isUnknownStream(error)) {
+    return { kind: 'rejected', reason: 'unknown-stream' };
+  }
+
+  if (isDefiniteRefusal(error)) {
+    return null;
+  }
+
+  return { kind: 'unknown' };
+}
+
+export function isUnknownStream(error: unknown): boolean {
+  return error instanceof S2Error && error.status === 404 && error.code === 'stream_not_found';
+}
+
+function isDefiniteRefusal(error: unknown): boolean {
+  return (
+    error instanceof S2Error &&
+    error.origin === 'server' &&
+    error.status >= 400 &&
+    error.status < 500 &&
+    error.status !== 408 &&
+    error.status !== 429
+  );
+}

package/src/index.ts

@@ -0,0 +1,23 @@
+export type * from './blob-store/blob-store.ts';
+export { createGarageBlobStore } from './blob-store/garage/create-garage-blob-store.ts';
+export { createLocalFolderBlobStore } from './blob-store/local-folder/create-local-folder-blob-store.ts';
+export type * from './fact-log/fact-log.ts';
+export { appendLimits } from './fact-log/check-append-batch.ts';
+export { makeBatchAppender } from './fact-log/make-batch-appender.ts';
+export type { AppendFailure, BatchAppender } from './fact-log/make-batch-appender.ts';
+export { createMemoryFactLog } from './fact-log/memory/create-memory-fact-log.ts';
+export type { MemoryFactLog } from './fact-log/memory/create-memory-fact-log.ts';
+export { planAppends } from './fact-log/plan-appends.ts';
+export { createS2FactLog } from './fact-log/s2/create-s2-fact-log.ts';
+export { createIngest } from './receive/create-ingest.ts';
+export type { Ingest, IngestOptions } from './receive/create-ingest.ts';
+export type { PullPathName, PullRun, PullRunner } from './pull/make-pull-runner.ts';
+export type { ResyncMarks } from './receive/resync-marks/create-resync-marks.ts';
+export { createFakeSenderProof } from './sender-proof/fake/create-fake-sender-proof.ts';
+export { createImpIdentitySenderProof } from './sender-proof/imp-identity/create-imp-identity-sender-proof.ts';
+export { readImpIdentitySenderProof } from './sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
+export { readImpIdentityTrust } from './sender-proof/imp-identity/read-imp-identity-trust.ts';
+export type { ImpIdentityTrust } from './sender-proof/imp-identity/types.ts';
+export { noSenderProof } from './sender-proof/none/no-sender-proof.ts';
+export { serveIngest } from './serve/serve-ingest.ts';
+export type * from './source-port/types.ts';

package/src/pull/make-pull-runner.ts

@@ -0,0 +1,104 @@
+import type { CheckedDeployment, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import { findBatchProblem } from '../receive/find-batch-problem.ts';
+import { planBatch } from '../receive/plan-batch.ts';
+import type { ResyncMarks } from '../receive/resync-marks/create-resync-marks.ts';
+import { writeBatch } from '../receive/write-batch.ts';
+import type { StreamTurns } from '../receive/write-batch.ts';
+
+export type PullPathName = 'live' | 'recovery' | 'backfill' | 'resync';
+
+// `stored`: the batch is in the log, and the next run starts from `cursor`. `not-stored`: an append
+// failed, so the next run starts from the same cursor. Any other answer is the pull's own.
+export type PullRun =
+  | { kind: 'stored'; stored: number; cursor: string; isDone: boolean }
+  | { kind: 'not-stored' }
+  | { kind: 'refused'; reason: string }
+  | Exclude<Pulled, { kind: 'batch' }>;
+
+export type PullRunner = (
+  source: string,
+  path: PullPathName,
+  cursor: string | null,
+) => Promise<PullRun>;
+
+interface PullRunnerDependencies {
+  deployment: CheckedDeployment;
+
+  // one value per name that a pull path's `env` lists
+  secrets: Readonly<Record<string, string>>;
+  append: BatchAppender;
+  turns: StreamTurns;
+  resyncMarks: ResyncMarks;
+}
+
+// One run of one pull path: the pull, the envelope check, the appends, and then, for a resync pull,
+// the removal of every mark it was given. A pull's fact is noted in no mark: a resynced fact ties
+// with the body it replaces, and would mark its entity again on every run.
+export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
+  return async (sourceName, pathName, cursor) => {
+    const source = dependencies.deployment.sources.find(
+      (candidate) => candidate.name === sourceName,
+    );
+
+    const path = source === undefined ? undefined : findPullPath(source, pathName);
+
+    if (source === undefined || path === undefined) {
+      return { kind: 'refused', reason: `the source ${sourceName} has no ${pathName} pull` };
+    }
+
+    const marks = pathName === 'resync' ? dependencies.resyncMarks.list(source.name) : [];
+    const env: Record<string, string> = {};
+
+    for (const name of path.env ?? []) {
+      env[name] = dependencies.secrets[name] ?? '';
+    }
+
+    const pulled = await path.pull(cursor, { env, marks });
+
+    if (pulled.kind !== 'batch') {
+      return pulled;
+    }
+
+    const problem = findBatchProblem(source, pulled.batch);
+
+    if (problem !== undefined) {
+      return { kind: 'refused', reason: problem };
+    }
+
+    if (pulled.batch.facts.some((pending) => pending.blobs.length > 0)) {
+      return { kind: 'refused', reason: 'ingest has no blob store' };
+    }
+
+    const plan = planBatch(pulled.batch);
+
+    if (!plan.ok) {
+      return { kind: 'refused', reason: 'one fact is larger than one append of the log takes' };
+    }
+
+    const outcome = await writeBatch(plan.value, dependencies);
+
+    if (outcome !== 'stored') {
+      return outcome === 'not-stored'
+        ? { kind: 'not-stored' }
+        : { kind: 'refused', reason: 'the log refused the batch' };
+    }
+
+    for (const mark of marks) {
+      dependencies.resyncMarks.remove(source.name, mark.entity);
+    }
+
+    return {
+      kind: 'stored',
+      stored: pulled.batch.facts.length,
+      cursor: pulled.cursor,
+      isDone: pulled.isDone,
+    };
+  };
+}
+
+function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
+  const path = source[pathName];
+
+  return path?.kind === 'pull' ? path : undefined;
+}

package/src/receive/answers/build-failure-answer.ts

@@ -0,0 +1,12 @@
+// Every failure body is `{"error": "<reason>"}`, on every path.
+export function buildFailureAnswer(status: number, reason: string): Response {
+  return Response.json({ error: reason }, { status });
+}
+
+// 503 means "send the same delivery again, unchanged": every sender of v1 does that.
+export function buildRetryAnswer(): Response {
+  return Response.json(
+    { error: 'the batch is not stored, send it again' },
+    { status: 503, headers: { 'retry-after': '1' } },
+  );
+}

package/src/receive/answers/build-success-answer.ts

@@ -0,0 +1,19 @@
+import type { ReceivePath } from '@pepys/kit';
+import { match } from 'ts-pattern';
+
+// 'status': 200 {"status":"stored","stored":<n>}, or 202 {"status":"ignored"}. 'empty': 200 {} for
+// both, which an OTLP exporter expects.
+export function buildSuccessAnswer(
+  path: ReceivePath,
+  outcome: 'stored' | 'ignored',
+  count: number,
+): Response {
+  return match(path.answer ?? 'status')
+    .with('empty', () => Response.json({}, { status: 200 }))
+    .with('status', () =>
+      outcome === 'stored'
+        ? Response.json({ status: 'stored', stored: count }, { status: 200 })
+        : Response.json({ status: 'ignored' }, { status: 202 }),
+    )
+    .exhaustive();
+}