@relayflows/sdk 2.0.0

package/src/demo-hn-monitor.ts

@@ -0,0 +1,132 @@
+import { readFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { pollHackerNewsOnce, type EventSink } from './hn-poller.js';
+import { JournalClient } from './journal-client.js';
+import type { EventSubmitResult } from './protocol.js';
+
+const sdkRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const repositoryRoot = resolve(sdkRoot, '..');
+const dataDir = resolve(process.env.RELAYFLOW_DATA_DIR ?? join(repositoryRoot, '.relayflowd'));
+const socketPath = join(dataDir, 'relayflowd.sock');
+const specPath = join(repositoryRoot, 'testdata', 'hn-monitor.spec.canonical.json');
+
+interface Submission {
+  storyId: unknown;
+  outcome: EventSubmitResult;
+}
+
+async function main(): Promise<void> {
+  const spec: unknown = JSON.parse(await readFile(specPath, 'utf8'));
+  const client = new JournalClient(socketPath);
+
+  try {
+    await client.connect();
+  } catch (error) {
+    throw new Error(
+      `No relayflowd is listening at "${socketPath}". Start it with: relayflowd --data-dir "${dataDir}" serve`,
+      { cause: error },
+    );
+  }
+
+  try {
+    await client.hello('hn-monitor-demo');
+    const submissions: Submission[] = [];
+    const sink: EventSink = {
+      async eventSubmit(submittedSpec, event) {
+        const outcome = await client.eventSubmit(submittedSpec, event);
+        submissions.push({ storyId: storyId(event.payload), outcome });
+        return outcome;
+      },
+    };
+
+    console.log('Fetching live Hacker News top stories...');
+    await pollHackerNewsOnce(spec, sink);
+
+    let woke = 0;
+    const createdRunIds: string[] = [];
+    for (const { storyId, outcome } of submissions) {
+      const wake = outcome.run === undefined || outcome.run === null ? 'none' : 'created';
+      if (wake === 'created') {
+        woke += 1;
+        const runId = typeof outcome.run === 'string' ? outcome.run : (outcome.run as { run_id?: string })?.run_id;
+        if (runId) createdRunIds.push(runId);
+      }
+      console.log(
+        `Story ${String(storyId)}: matched=${outcome.matched} deduped=${outcome.deduped} wake=${wake}`,
+      );
+    }
+
+    // Say exactly what was proven, and no more.
+    //
+    // A wake is a run being CREATED. It is not the flow's steps being
+    // EXECUTED — that needs an agent worker attached to the kernel, and
+    // `relayflowd serve` alone does not provide one. Review caught this
+    // (PR #19, P1): the first version of this demo printed created wakes and
+    // let the reader conclude a workload had run. It had not.
+    //
+    // The distinction is the whole of RFC-0001 §3 gate 2: "a real proactive
+    // workload RUNS as a relayflow" is a claim about execution, not about
+    // enqueueing.
+    console.log('');
+    if (woke > 0) {
+      console.log(`PROVEN: ${woke} run(s) created from live Hacker News data via event.submit.`);
+      console.log('        The event path — fetch, match, dedupe claim, wake — works end to end.');
+      console.log('');
+
+      // Do not ASSERT that nothing executed — ask the kernel and report what it
+      // says. The first version of this block hardcoded "no agent worker is
+      // attached", which would have been a false statement the moment someone
+      // attached one. Evidence is captured, not narrated.
+      const observed = createdRunIds[0];
+      if (observed === undefined) {
+        console.log('NOT PROVEN: that those runs EXECUTED — no run id came back to inspect.');
+      } else {
+        const snapshot = await client.runGet(observed);
+        const steps = Object.entries(snapshot.steps);
+        // A run nobody works sits in `runnable` — ready, with no worker to
+        // claim it. So "not pending" is NOT evidence of execution; only a step
+        // that reached `running` or `done` proves a worker picked it up.
+        const executed = steps.filter(([, step]) => step.state === 'running' || step.state === 'done').length;
+        const stateCounts = steps.map(([id, step]) => `${id}=${step.state}`).join(' ');
+        console.log(`Observed run ${observed}: status=${snapshot.status}, steps: ${stateCounts}`);
+        if (executed === 0) {
+          console.log('');
+          console.log('NOT PROVEN: that those runs EXECUTED. No step reached running or done,');
+          console.log('        which is what a created-but-unworked run looks like: `relayflowd serve`');
+          console.log('        alone attaches no agent worker. Gate 2 asks whether a workload RUNS');
+          console.log('        as a relayflow; this shows it is woken, not that it ran.');
+          // Ordering matters, and the obvious advice is wrong. A run that
+          // finds no worker parks; attaching one AFTERWARDS does not re-drive
+          // it, because nothing revisits parked runs. Measured on the live
+          // kernel: attach-then-submit dispatches, submit-then-attach does not
+          // until run.resume is called.
+          console.log('        To close it, the worker must be attached BEFORE these events are');
+          console.log('        submitted — attaching afterwards does not re-drive a parked run.');
+          console.log('        Already parked? Call run.resume on it once a worker is attached.');
+        } else {
+          console.log('');
+          console.log(`ALSO PROVEN: execution happened — ${String(executed)} step(s) reached`);
+          console.log('        running or done, so a worker claimed this run. That is gate 2 proper.');
+        }
+      }
+    } else {
+      console.log('No runs were created. Either every story was already claimed (dedupe working');
+      console.log('as intended on a repeat poll), or nothing matched the subscription.');
+    }
+  } finally {
+    client.close();
+  }
+}
+
+function storyId(payload: unknown): unknown {
+  if (typeof payload === 'object' && payload !== null && 'id' in payload) {
+    return payload.id;
+  }
+  return 'unknown';
+}
+
+main().catch((error: unknown) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exitCode = 1;
+});

package/src/dir-watcher-poller.ts

@@ -0,0 +1,122 @@
+/**
+ * Directory watcher -> relayflow events.
+ *
+ * Second proactive workload on gate 2 primitives (hn-monitor is the first).
+ * Deliberately non-provider: no HTTP, no API tokens, no gate-6 dependency —
+ * just a directory poll. This proves the runner pattern generalizes beyond
+ * `hn-poller` without regressing RFC-0001 §6 (which assigns providers to
+ * relayfile adapters, not SDK code).
+ *
+ * How it works: each poll lists the target directory, dedupes against a
+ * caller-supplied `seen` set (or an internal Map if none provided), and
+ * submits a `dir.file_appeared` event for each unseen entry through the
+ * journal protocol. The kernel then dispatches the flow's agent step for
+ * each new file.
+ *
+ * Deduplication is still ultimately the kernel's job (flow's
+ * `dedupeKeyTemplate` + the (flow, subscription, key) claim). This layer's
+ * `seen` set is a cheap pre-filter so we don't spam `event.submit` with the
+ * same paths on every poll — an optimization, not a correctness contract.
+ *
+ * Real-world analog: an "inbox" directory that a human or another system
+ * drops files into, triggering a per-file flow (summarize, ingest, route,
+ * whatever the step declares).
+ */
+
+import { promises as fsp } from 'node:fs';
+import { join } from 'node:path';
+
+/** Anything that can submit an event through the journal protocol. */
+export interface EventSink {
+  eventSubmit(spec: unknown, event: { type: string; payload?: unknown; key?: string }): Promise<unknown>;
+}
+
+/** Injected so I/O stays deterministic in tests. */
+export interface DirLister {
+  (dir: string): Promise<Array<{ name: string; size: number; mtimeMs: number; isFile: boolean }>>;
+}
+
+const defaultLister: DirLister = async (dir) => {
+  const entries = await fsp.readdir(dir, { withFileTypes: true });
+  const out: Array<{ name: string; size: number; mtimeMs: number; isFile: boolean }> = [];
+  for (const ent of entries) {
+    if (!ent.isFile()) continue;
+    const full = join(dir, ent.name);
+    const stat = await fsp.stat(full);
+    out.push({
+      name: ent.name,
+      size: stat.size,
+      mtimeMs: stat.mtimeMs,
+      isFile: true,
+    });
+  }
+  return out;
+};
+
+export interface PollOptions {
+  /** Directory to watch. Required. */
+  dir: string;
+  /**
+   * Set of relative paths already seen. The poller mutates it, adding each
+   * new file it submits. Callers persist this across polls to avoid
+   * re-submitting; internal callers can pass a fresh Set each poll if
+   * they'd rather rely on the kernel's dedupe claim.
+   */
+  seen: Set<string>;
+  /**
+   * Lister override — tests inject a deterministic fake. Production uses
+   * fs.readdir.
+   */
+  lister?: DirLister;
+  /**
+   * Cap on files per poll (safety valve against dropping thousands into
+   * the directory at once). Default 100.
+   */
+  fileLimit?: number;
+}
+
+const DEFAULT_FILE_LIMIT = 100;
+
+/**
+ * List the directory once and submit a `dir.file_appeared` event for each
+ * unseen file. Adds each submitted path to `seen`.
+ *
+ * Returns the submit outcomes (one per new file). Journal errors from
+ * `eventSubmit` propagate; empty result is not an error; a missing
+ * directory throws (the caller decides whether that's a fetch error or
+ * a real failure — the runner classifies).
+ */
+export async function pollDirectoryOnce(
+  spec: unknown,
+  sink: EventSink,
+  options: PollOptions,
+): Promise<unknown[]> {
+  const lister = options.lister ?? defaultLister;
+  const fileLimit = options.fileLimit ?? DEFAULT_FILE_LIMIT;
+  const seen = options.seen;
+
+  const entries = await lister(options.dir);
+  const fresh = entries
+    .filter((e) => e.isFile && !seen.has(e.name))
+    .slice(0, fileLimit);
+
+  const outcomes: unknown[] = [];
+  for (const entry of fresh) {
+    outcomes.push(
+      await sink.eventSubmit(spec, {
+        type: 'dir.file_appeared',
+        payload: {
+          type: 'file',
+          path: entry.name,
+          size: entry.size,
+          mtime_ms: entry.mtimeMs,
+        },
+      }),
+    );
+    // Only add to `seen` AFTER a successful submit — a journal failure
+    // means the event didn't reach the kernel, so the next poll should
+    // retry submission.
+    seen.add(entry.name);
+  }
+  return outcomes;
+}

package/src/direct-input.ts

@@ -0,0 +1,69 @@
+import { readFileSync, statSync } from 'node:fs';
+import { resolve } from 'node:path';
+import type { CheckFailureKind } from './failure-kinds.js';
+
+export const MAX_DIRECT_INPUT_BYTES = 1_048_576;
+
+export class DirectInputError extends Error {
+  constructor(readonly kind: CheckFailureKind, message: string) {
+    super(message);
+    this.name = 'DirectInputError';
+  }
+}
+
+/** Parse a direct-run input as an existing JSON file, otherwise as inline JSON. */
+export function parseDirectInput(argument: string | undefined): unknown {
+  if (argument === undefined) {
+    throw new DirectInputError(
+      'input_missing',
+      'A directly run .flow.ts requires --input <inline-json-or-file>.',
+    );
+  }
+
+  const inputPath = resolve(argument);
+  let source = argument;
+  let fromFile = false;
+  try {
+    const stat = statSync(inputPath);
+    if (!stat.isFile()) {
+      throw new DirectInputError('input_unreadable', `Input file "${argument}" is not a regular file.`);
+    }
+    if (stat.size > MAX_DIRECT_INPUT_BYTES) {
+      throw tooLarge(argument, true);
+    }
+    try {
+      source = readFileSync(inputPath, 'utf8');
+      fromFile = true;
+    } catch {
+      throw new DirectInputError('input_unreadable', `Input file "${argument}" is not readable.`);
+    }
+  } catch (error) {
+    if (error instanceof DirectInputError) throw error;
+    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+      throw new DirectInputError('input_unreadable', `Input file "${argument}" could not be inspected.`);
+    }
+  }
+
+  if (!fromFile && Buffer.byteLength(source, 'utf8') > MAX_DIRECT_INPUT_BYTES) {
+    throw tooLarge(argument, false);
+  }
+
+  try {
+    return JSON.parse(source) as unknown;
+  } catch {
+    const sourceKind = fromFile ? `Input file "${argument}"` : 'Inline input';
+    throw new DirectInputError('input_invalid', `${sourceKind} is not valid JSON.`);
+  }
+}
+
+function tooLarge(argument: string, fromFile: boolean): DirectInputError {
+  const sourceKind = fromFile ? `Input file "${argument}"` : 'Inline input';
+  return new DirectInputError(
+    'input_too_large',
+    `${sourceKind} exceeds the ${MAX_DIRECT_INPUT_BYTES}-byte direct input limit.`,
+  );
+}
+
+export function isAuthoredFlowPath(path: string): boolean {
+  return /\.flow\.(?:ts|mts|js|mjs)$/.test(path);
+}

package/src/failure-kinds.ts

@@ -0,0 +1,77 @@
+const SHARED_SPEC_FAILURE_KINDS = ['invalid_spec'] as const;
+
+/** Environment refusal kinds produced after spec validation succeeds. */
+const PREFLIGHT_ENVIRONMENT_FAILURE_KINDS = [
+  'cli_missing',
+  'cli_unauthenticated',
+  'cli_unresolved',
+  'cli_unsupported',
+  'command_missing',
+  'model_unavailable',
+  'model_unknown',
+  'no_executor',
+  'probe_failed',
+] as const;
+
+/** Closed refusal taxonomy for public preflight (RFC covenant 2). */
+export const PREFLIGHT_FAILURE_KINDS = [
+  ...SHARED_SPEC_FAILURE_KINDS,
+  ...PREFLIGHT_ENVIRONMENT_FAILURE_KINDS,
+] as const;
+
+/** Input/command refusals emitted before the pure preflight predicates run. */
+export const CHECK_INPUT_FAILURE_KINDS = [
+  'config_invalid',
+  'input_invalid',
+  'input_missing',
+  'input_too_large',
+  'input_unreadable',
+  'invalid_invocation',
+  ...SHARED_SPEC_FAILURE_KINDS,
+] as const;
+
+export const CHECK_FAILURE_KINDS = [
+  ...CHECK_INPUT_FAILURE_KINDS,
+  ...PREFLIGHT_ENVIRONMENT_FAILURE_KINDS,
+] as const;
+
+/**
+ * Warnings never refuse. Covenant 2 asks preflight to refuse *or warn* on
+ * anything it cannot prove, so a deterministic step always leaves exactly one
+ * of these: its command resolved (effects still unknowable), it did not
+ * resolve, or it could not be probed at all. Silence is not one of the states.
+ *
+ * `vacuous_gate` is the same principle applied to a declared gate that judges
+ * nothing: `schema: {}` and `schema: true` are legal and accepted, but a gate
+ * accepting every output must not be reported as if it constrained one.
+ */
+export const PREFLIGHT_WARNING_KINDS = [
+  'unprovable_effects',
+  'command_unresolved',
+  'command_unprovable',
+  'vacuous_gate',
+] as const;
+
+/** Closed outcome taxonomy owned by the `flows run` / `flows resume` surface. */
+export const RUN_FAILURE_KINDS = [
+  'daemon_unreachable',
+  'protocol_error',
+  'run_parked',
+  'run_unavailable',
+] as const;
+
+export type PreflightFailureKind = (typeof PREFLIGHT_FAILURE_KINDS)[number];
+export type CheckFailureKind = (typeof CHECK_FAILURE_KINDS)[number];
+export type PreflightWarningKind = (typeof PREFLIGHT_WARNING_KINDS)[number];
+export type RunFailureKind = (typeof RUN_FAILURE_KINDS)[number];
+
+const CHECK_FAILURE_KIND_SET: ReadonlySet<string> = new Set(CHECK_FAILURE_KINDS);
+const RUN_FAILURE_KIND_SET: ReadonlySet<string> = new Set(RUN_FAILURE_KINDS);
+
+export function isCheckFailureKind(value: string): value is CheckFailureKind {
+  return CHECK_FAILURE_KIND_SET.has(value);
+}
+
+export function isRunFailureKind(value: string): value is RunFailureKind {
+  return RUN_FAILURE_KIND_SET.has(value);
+}

package/src/gate-contract.ts

@@ -0,0 +1,72 @@
+import type { StepSpec } from './spec.js';
+
+export type JournalGateCheck =
+  | 'completion'
+  | 'exit_code'
+  | 'output_contains'
+  | 'json_schema';
+
+export interface DataGateClassification {
+  kind: 'data';
+  checks: JournalGateCheck[];
+  evaluator: 'kernel';
+  preflightable: true;
+  replayable: true;
+}
+
+export interface StepGateInspection extends DataGateClassification {
+  stepId: string;
+  /**
+   * Present only when the declared `json_schema` accepts every possible
+   * output. `{}` and `true` are legal JSON Schema and the kernel accepts both,
+   * so this is not a refusal — but a gate that judges nothing must not read
+   * identically to a gate that judges something.
+   */
+  acceptsAnyOutput?: true;
+}
+
+/** Annotation-only keywords: present, they still constrain no instance. */
+const ANNOTATIONS: ReadonlySet<string> = new Set([
+  '$anchor',
+  '$comment',
+  '$defs',
+  '$dynamicAnchor',
+  '$id',
+  '$schema',
+  '$vocabulary',
+  'default',
+  'definitions',
+  'deprecated',
+  'description',
+  'examples',
+  'readOnly',
+  'title',
+  'writeOnly',
+]);
+
+/** A schema that accepts every output: `true`, `{}`, or annotations only. */
+export function acceptsAnyOutput(schema: unknown): boolean {
+  if (schema === true) return true;
+  if (typeof schema !== 'object' || schema === null || Array.isArray(schema)) return false;
+  return Object.keys(schema).every((key) => ANNOTATIONS.has(key));
+}
+
+/** Describe the exact named checks the existing kernel applies to a step. */
+export function inspectStepGate(step: StepSpec): StepGateInspection {
+  const checks: JournalGateCheck[] = [];
+  if (step.type === 'deterministic') checks.push('exit_code');
+  if (step.verification?.type === 'output_contains') checks.push('output_contains');
+  if (step.verification?.type === 'json_schema') checks.push('json_schema');
+  if (checks.length === 0) checks.push('completion');
+  const vacuous = step.verification?.type === 'json_schema'
+    && acceptsAnyOutput(step.verification.schema);
+  return {
+    stepId: step.id,
+    kind: 'data',
+    checks,
+    evaluator: 'kernel',
+    preflightable: true,
+    replayable: true,
+    ...(vacuous ? { acceptsAnyOutput: true as const } : {}),
+  };
+}

package/src/hn-poller.ts

@@ -0,0 +1,104 @@
+/**
+ * Hacker News -> relayflow events.
+ *
+ * This lives OUTSIDE `kernel/` deliberately. An earlier version called Hacker
+ * News from `kernel/relayflowd` and review rejected it (PR #16, P1): a
+ * durable-execution kernel must not own provider-specific product logic or
+ * network I/O, or engine availability and dependencies become coupled to an
+ * external service. The kernel gained a `ureq` dependency purely to fetch a
+ * JSON feed — a clear sign the code was in the wrong place.
+ *
+ * So the adapter sits on the authoring surface and submits its events through
+ * the journal protocol (`event.submit`), which is the same path any other
+ * external producer would use. The kernel learns about Hacker News the way it
+ * learns about everything else: as an event.
+ */
+
+const TOP_STORIES_URL = 'https://hacker-news.firebaseio.com/v0/topstories.json';
+const DEFAULT_STORY_LIMIT = 5;
+
+/** Anything that can submit an event through the journal protocol. */
+export interface EventSink {
+  eventSubmit(spec: unknown, event: { type: string; payload?: unknown; key?: string }): Promise<unknown>;
+}
+
+/** Injected so parsing and submission stay deterministic in tests. */
+export type Fetcher = (url: string) => Promise<string>;
+
+/**
+ * Typed fetch-transport error. Callers (like sdk/src/cli/hn-monitor.ts)
+ * `instanceof` this to distinguish transient HN fetch failures from
+ * journal failures — avoids message-string matching (fragile cross-module
+ * coupling) and covers the shapes fetch() itself throws (network errors,
+ * ECONNREFUSED, TypeError('fetch failed')).
+ */
+export class HnTransientFetchError extends Error {
+  constructor(message: string, cause?: unknown) {
+    // Use the native ErrorOptions.cause path so stack formatting and
+    // downstream inspectors (util.inspect, structured loggers) see it.
+    super(message, cause !== undefined ? { cause } : undefined);
+    this.name = 'HnTransientFetchError';
+  }
+}
+
+const defaultFetcher: Fetcher = async (url) => {
+  let response: Response;
+  try {
+    response = await fetch(url);
+  } catch (cause) {
+    // Network-layer failures from fetch() itself — TypeError('fetch failed'),
+    // ECONNREFUSED, DNS lookup failures — are all transient.
+    throw new HnTransientFetchError(`HN fetch failed: ${String(cause)}`, cause);
+  }
+  if (!response.ok) {
+    throw new HnTransientFetchError(`HN fetch failed: HTTP ${response.status}`);
+  }
+  return response.text();
+};
+
+export interface PollOptions {
+  storyLimit?: number;
+  fetcher?: Fetcher;
+  createdBy?: string;
+}
+
+/**
+ * Fetch the top-stories feed once and submit each story as an event.
+ *
+ * Dedupe is the kernel's job, not ours: the flow's `dedupeKeyTemplate` plus the
+ * (flow, subscription, key) claim means submitting the same story twice wakes
+ * it once. This function deliberately does not track what it has already seen.
+ */
+export async function pollHackerNewsOnce(
+  spec: unknown,
+  sink: EventSink,
+  options: PollOptions = {},
+): Promise<unknown[]> {
+  const storyLimit = options.storyLimit ?? DEFAULT_STORY_LIMIT;
+  const fetcher = options.fetcher ?? defaultFetcher;
+
+  const body = await fetcher(TOP_STORIES_URL);
+
+  let storyIds: unknown;
+  try {
+    storyIds = JSON.parse(body);
+  } catch (cause) {
+    throw new HnTransientFetchError(`HN top stories response was not JSON: ${String(cause)}`);
+  }
+  if (!Array.isArray(storyIds)) {
+    throw new HnTransientFetchError('HN top stories response was not an array');
+  }
+
+  const outcomes: unknown[] = [];
+  for (const id of storyIds.slice(0, storyLimit)) {
+    outcomes.push(
+      await sink.eventSubmit(spec, {
+        type: 'hn.story_posted',
+        payload: { id, type: 'story' },
+      }),
+    );
+  }
+  return outcomes;
+}
+
+export const HN_TOP_STORIES_URL = TOP_STORIES_URL;