@wringer/stream 0.0.0-stage → 0.1.0

package/src/read-stream-slice.ts

@@ -0,0 +1,84 @@
+import type { FileHandle } from 'node:fs/promises';
+import { WringerError, isErrorCode } from '@wringer/core';
+import { buildStreamSlice } from './build-stream-slice.ts';
+import type { StreamSlice } from './build-stream-slice.ts';
+import type { OpenFile } from './types.ts';
+
+export interface StreamSliceRequest {
+  readonly path: string;
+  readonly from: number;
+  readonly maxBytes: number;
+}
+
+// Reads whole records from the byte offset `from`, up to `maxBytes`. A file the machine refuses to
+// read, such as a folder at the stream's path, is a fault of the environment, not of wringer.
+export async function readStreamSlice(
+  request: StreamSliceRequest,
+  openFile: OpenFile,
+): Promise<StreamSlice> {
+  try {
+    return await readSlice(request, openFile);
+  } catch (error) {
+    throw new WringerError('environment', {
+      message: `The stream file ${request.path} could not be read: ${String(error)}`,
+    });
+  }
+}
+
+// The recorder caps each record well under the smallest slice, so a full slice always ends on a
+// line end. A cursor past the end of the file comes from an older run, so the read starts over at 0.
+async function readSlice(request: StreamSliceRequest, openFile: OpenFile): Promise<StreamSlice> {
+  const handle = await openStreamFile(openFile, request.path);
+
+  if (handle === null) {
+    return { records: [], cursor: 0, more: false };
+  }
+
+  try {
+    const info = await handle.stat();
+
+    const size = info.size;
+    const requested = request.from > size ? 0 : request.from;
+
+    const from = await readLineStart(handle, requested, size);
+
+    const length = Math.min(request.maxBytes, size - from);
+    const buffer = Buffer.alloc(length);
+
+    await handle.read(buffer, 0, length, from);
+
+    return buildStreamSlice(buffer, from, size);
+  } finally {
+    await handle.close();
+  }
+}
+
+// A stream that has recorded nothing yet has no file. Any other failure, such as EACCES, rejects.
+async function openStreamFile(openFile: OpenFile, path: string): Promise<FileHandle | null> {
+  try {
+    return await openFile(path);
+  } catch (error) {
+    if (isErrorCode(error, 'ENOENT')) {
+      return null;
+    }
+
+    throw error;
+  }
+}
+
+const newline = 0x0a;
+
+// A cursor a caller typed may land inside a record; the read then starts at the next one.
+async function readLineStart(handle: FileHandle, from: number, size: number): Promise<number> {
+  if (from === 0) {
+    return 0;
+  }
+
+  const buffer = Buffer.alloc(Math.min(size - from + 1, 64 * 1024));
+
+  await handle.read(buffer, 0, buffer.length, from - 1);
+
+  const index = buffer.indexOf(newline);
+
+  return index === -1 ? size : from + index;
+}

package/src/record-writer-machine.ts

@@ -0,0 +1,185 @@
+import { WringerError } from '@wringer/core';
+import { assign, fromPromise, setup } from 'xstate';
+
+interface WriterInput {
+  readonly path: string;
+  readonly pendingLimit: number;
+  readonly now: () => Date;
+  readonly write: (text: string) => Promise<void>;
+  readonly onError: (error: unknown) => void;
+
+  // Called when a write succeeds after one that failed.
+  readonly onRecovered: () => void;
+}
+
+interface WriterContext extends WriterInput {
+  readonly pending: readonly string[];
+  readonly batch: readonly string[];
+  readonly batchDropped: number;
+  readonly dropped: number;
+  readonly failure: string | null;
+  readonly drains: readonly PromiseWithResolvers<null>[];
+}
+
+type WriterEvent =
+  | { readonly type: 'add'; readonly record: string }
+  | { readonly type: 'drain'; readonly settle: PromiseWithResolvers<null> };
+
+const writerTypes: { context?: WriterContext; events?: WriterEvent; input?: WriterInput } = {};
+
+interface BatchInput {
+  readonly write: WriterInput['write'];
+  readonly lines: readonly string[];
+}
+
+// Appends the records that arrived while the last write ran in one write, so a fast source costs
+// one write per batch and a bounded amount of memory. A batch that fails to write goes back to the
+// front of the queue: the next record retries it, and a drain retries it once, then rejects.
+export const recordWriterMachine = setup({
+  types: writerTypes,
+  actors: {
+    writeBatch: fromPromise<null, BatchInput>(async (args) => {
+      await args.input.write(`${args.input.lines.join('\n')}\n`);
+
+      return null;
+    }),
+  },
+  guards: {
+    hasPending: (args) => args.context.pending.length > 0 || args.context.dropped > 0,
+  },
+  actions: {
+    enqueue: assign((args) => {
+      if (args.event.type !== 'add') {
+        return {};
+      }
+
+      return args.context.pending.length >= args.context.pendingLimit
+        ? { dropped: args.context.dropped + 1 }
+        : { pending: [...args.context.pending, args.event.record] };
+    }),
+    takeBatch: assign((args) => ({
+      batch: args.context.pending,
+      batchDropped: args.context.dropped,
+      pending: [],
+      dropped: 0,
+    })),
+    keepBatch: assign((args) => {
+      const kept = [...args.context.batch, ...args.context.pending];
+      const limit = args.context.pendingLimit;
+
+      return {
+        batch: [],
+        batchDropped: 0,
+        pending: kept.slice(0, limit),
+        dropped:
+          args.context.batchDropped + args.context.dropped + Math.max(kept.length - limit, 0),
+      };
+    }),
+    keepDrain: assign({
+      drains: (args) =>
+        args.event.type === 'drain'
+          ? [...args.context.drains, args.event.settle]
+          : args.context.drains,
+    }),
+    resolveDrains: assign((args) => {
+      for (const drain of args.context.drains) {
+        drain.resolve(null);
+      }
+
+      return { drains: [] };
+    }),
+    reportRecovery: (args) => {
+      if (args.context.failure !== null) {
+        args.context.onRecovered();
+      }
+    },
+    rejectDrains: assign((args) => {
+      const error = new WringerError('environment', {
+        message: `${args.context.path} kept ${args.context.pending.length} records the recorder could not write: ${args.context.failure ?? 'the write failed'}`,
+      });
+
+      for (const drain of args.context.drains) {
+        drain.reject(error);
+      }
+
+      return { drains: [] };
+    }),
+  },
+}).createMachine({
+  context: (args) => ({
+    ...args.input,
+    pending: [],
+    batch: [],
+    batchDropped: 0,
+    dropped: 0,
+    failure: null,
+    drains: [],
+  }),
+  initial: 'idle',
+  states: {
+    idle: {
+      entry: 'resolveDrains',
+      on: {
+        add: { target: 'writing', actions: 'enqueue' },
+        drain: { actions: ['keepDrain', 'resolveDrains'] },
+      },
+    },
+    writing: {
+      entry: 'takeBatch',
+      invoke: {
+        // oxlint-disable-next-line zgeoff/no-abbreviated-names -- xstate names this key `src`
+        src: 'writeBatch',
+        input: (args) => ({
+          write: args.context.write,
+          lines: [
+            ...args.context.batch,
+            ...buildDropNotice(args.context.batchDropped, args.context.now),
+          ],
+        }),
+        onDone: [
+          {
+            guard: 'hasPending',
+            target: 'writing',
+            reenter: true,
+            actions: ['reportRecovery', assign({ batch: [], batchDropped: 0, failure: null })],
+          },
+          {
+            target: 'idle',
+            actions: ['reportRecovery', assign({ batch: [], batchDropped: 0, failure: null })],
+          },
+        ],
+        onError: {
+          target: 'stalled',
+          actions: [
+            'keepBatch',
+            assign({
+              failure: (args) =>
+                args.event.error instanceof Error ? args.event.error.message : 'the write failed',
+            }),
+            (args) => {
+              args.context.onError(args.event.error);
+            },
+          ],
+        },
+      },
+      on: { add: { actions: 'enqueue' }, drain: { actions: 'keepDrain' } },
+    },
+    stalled: {
+      entry: 'rejectDrains',
+      on: {
+        add: { target: 'writing', actions: 'enqueue' },
+        drain: { target: 'writing', actions: 'keepDrain' },
+      },
+    },
+  },
+});
+
+function buildDropNotice(dropped: number, now: () => Date): string[] {
+  if (dropped === 0) {
+    return [];
+  }
+
+  const line = `[wringer] the recorder dropped ${dropped} lines while it caught up`;
+
+  return [JSON.stringify({ at: now().toISOString(), line })];
+}

package/src/source-phase-machine.ts

@@ -0,0 +1,132 @@
+import { assign, setup } from 'xstate';
+
+// A source's phase joins how its reading ended with whether its last write failed. A failed write
+// shows as `failed` until a write succeeds, then the source shows how its reading stands again.
+export type SourcePhase =
+  | 'running'
+  | 'running-unwritten'
+  | 'exited'
+  | 'exited-unwritten'
+  | 'failed'
+  | 'failed-unwritten';
+
+export type SourceEventType = 'exit' | 'fail' | 'write-failed' | 'write-recovered';
+
+// `detail` is why the reading ended for `exit` and `fail`, and why the write failed for
+// `write-failed`.
+export interface SourceEvent {
+  readonly type: SourceEventType;
+  readonly detail: string | null;
+}
+
+export interface SourceDetails {
+  // Why the reading ended, once it has.
+  readonly reading: string | null;
+  readonly write: string | null;
+}
+
+interface SourceContext {
+  readonly phase: SourcePhase;
+  readonly details: SourceDetails;
+}
+
+const sourceTypes: { context?: SourceContext; events?: SourceEvent } = {};
+
+// The detail each event sets once it moves the source.
+const eventDetails: Readonly<Record<SourceEventType, keyof SourceDetails>> = {
+  exit: 'reading',
+  fail: 'reading',
+  'write-failed': 'write',
+  'write-recovered': 'write',
+};
+
+// Holds a source's phase and the details its status reports. An event the transition table refuses
+// leaves both as they are, so the first reason a reading ended stays.
+export const sourcePhaseMachine = setup({
+  types: sourceTypes,
+  actions: {
+    move: assign((args) => {
+      const next = pickSourcePhase(args.context.phase, args.event.type);
+
+      if (next === args.context.phase) {
+        return {};
+      }
+
+      return {
+        phase: next,
+        details: { ...args.context.details, [eventDetails[args.event.type]]: args.event.detail },
+      };
+    }),
+  },
+}).createMachine({
+  context: { phase: 'running', details: { reading: null, write: null } },
+  on: {
+    exit: { actions: 'move' },
+    fail: { actions: 'move' },
+    'write-failed': { actions: 'move' },
+    'write-recovered': { actions: 'move' },
+  },
+});
+
+// The supervisor sends each source's status to the CLI, so every change of it goes through this
+// table.
+const sourceTransitions: Readonly<Record<SourcePhase, readonly SourcePhase[]>> = {
+  running: ['running-unwritten', 'exited', 'failed'],
+  'running-unwritten': ['running', 'exited-unwritten', 'failed-unwritten'],
+  exited: ['exited-unwritten'],
+  'exited-unwritten': ['exited'],
+  failed: ['failed-unwritten'],
+  'failed-unwritten': ['failed'],
+};
+
+export function canMoveSource(from: SourcePhase, to: SourcePhase): boolean {
+  return sourceTransitions[from].includes(to);
+}
+
+// The phase each event aims at; the table above decides whether the source may move there.
+const eventTargets: Readonly<Record<SourcePhase, Readonly<Record<SourceEventType, SourcePhase>>>> =
+  {
+    running: {
+      exit: 'exited',
+      fail: 'failed',
+      'write-failed': 'running-unwritten',
+      'write-recovered': 'running',
+    },
+    'running-unwritten': {
+      exit: 'exited-unwritten',
+      fail: 'failed-unwritten',
+      'write-failed': 'running-unwritten',
+      'write-recovered': 'running',
+    },
+    exited: {
+      exit: 'exited',
+      fail: 'failed',
+      'write-failed': 'exited-unwritten',
+      'write-recovered': 'exited',
+    },
+    'exited-unwritten': {
+      exit: 'exited-unwritten',
+      fail: 'failed-unwritten',
+      'write-failed': 'exited-unwritten',
+      'write-recovered': 'exited',
+    },
+    failed: {
+      exit: 'exited',
+      fail: 'failed',
+      'write-failed': 'failed-unwritten',
+      'write-recovered': 'failed',
+    },
+    'failed-unwritten': {
+      exit: 'exited-unwritten',
+      fail: 'failed-unwritten',
+      'write-failed': 'failed-unwritten',
+      'write-recovered': 'failed',
+    },
+  };
+
+// An event the table refuses from this phase, such as an exit after a failure, leaves it as it is.
+export function pickSourcePhase(phase: SourcePhase, event: SourceEventType): SourcePhase {
+  const target = eventTargets[phase][event];
+
+  return canMoveSource(phase, target) ? target : phase;
+}