@llm4ts/flow 0.1.0

package/src/Pack.ts

@@ -0,0 +1,213 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { Dimension } from "@llm4ts/core/eval/Eval"
+import { PlanParseError } from "./FlowError.ts"
+import { parseReviewer, type Reviewer } from "./Reviewer.ts"
+import { CoverageRule } from "./SpecChecks.ts"
+import type { WorkspaceError, WorkspaceShape } from "./Workspace.ts"
+
+export { Reviewer, parseReviewer } from "./Reviewer.ts"
+
+export const ComparisonOrdering = Schema.Literals(["Ordered", "Unordered", "PerKey"])
+export type ComparisonOrdering = typeof ComparisonOrdering.Type
+
+export class ComparisonPolicy extends Schema.Class<ComparisonPolicy>("ComparisonPolicy")({
+  ordering: ComparisonOrdering,
+  ignore: Schema.ReadonlySet(Schema.String)
+}) {}
+
+export interface Pack {
+  readonly name: string
+  readonly source: string
+  readonly scaffold: string | undefined
+  readonly sources: string | undefined
+  readonly programs: string | undefined
+  readonly specsDir: string
+  readonly featuresDir: string
+  readonly gates: Readonly<Record<string, ReadonlyArray<string>>>
+  readonly replay: ReadonlyArray<string> | undefined
+  readonly equivalence: ComparisonPolicy
+  readonly judgeDimensions: ReadonlyArray<Dimension>
+  readonly coverage: ReadonlyArray<CoverageRule>
+  readonly survey: ReadonlyArray<CoverageRule>
+  readonly prompts: Readonly<Record<string, string>>
+  readonly lenses: ReadonlyArray<Reviewer>
+  readonly lessons: string | undefined
+  readonly dir: string
+  readonly gate: (name: string) => ReadonlyArray<string> | undefined
+  readonly prompt: (name: string) => string | undefined
+}
+
+interface ParsedManifest {
+  readonly name: string
+  readonly fields: Readonly<Record<string, string>>
+  readonly sections: ReadonlyArray<string>
+}
+
+const parseManifest = (markdown: string): Effect.Effect<ParsedManifest, PlanParseError> => {
+  const chunks = markdown
+    .trim()
+    .split(/^(?=## )/m)
+    .map((chunk) => chunk.trim())
+    .filter((chunk) => chunk.length > 0)
+  const head = chunks[0]
+  const prefix = "# Pack: "
+  if (head === undefined || !head.startsWith(prefix)) {
+    return Effect.fail(
+      PlanParseError.make({
+        message: `expected a '${prefix}<name>' header`
+      })
+    )
+  }
+  const lines = head.split(/\r?\n/)
+  const fields = Object.fromEntries(
+    lines.slice(1).flatMap((line) => {
+      const index = line.indexOf(":")
+      return index < 0 || line.trim().startsWith("#") || line.trim().startsWith("-")
+        ? []
+        : [[line.slice(0, index).trim(), line.slice(index + 1).trim()]]
+    })
+  )
+  return fields.source === undefined
+    ? Effect.fail(
+        PlanParseError.make({
+          message: "pack manifest is missing a 'source:' field"
+        })
+      )
+    : Effect.succeed({
+        name: lines[0]?.slice(prefix.length).trim() ?? "",
+        fields,
+        sections: chunks.slice(1)
+      })
+}
+
+const section = (sections: ReadonlyArray<string>, title: string): string | undefined =>
+  sections
+    .find((candidate) => candidate.split(/\r?\n/)[0]?.trim() === `## ${title}`)
+    ?.split(/\r?\n/)
+    .slice(1)
+    .join("\n")
+    .trim()
+
+const namedItems = (body: string | undefined): Readonly<Record<string, string>> =>
+  Object.fromEntries(
+    (body ?? "").split(/\r?\n/).flatMap((line) => {
+      const match = /^- ([^:]+): (.+)$/.exec(line)
+      return match?.[1] === undefined || match[2] === undefined
+        ? []
+        : [[match[1].trim(), match[2].trim()]]
+    })
+  )
+
+const rules = (sections: ReadonlyArray<string>, prefix: string): ReadonlyArray<CoverageRule> =>
+  sections.flatMap((candidate) => {
+    const lines = candidate.split(/\r?\n/)
+    const heading = lines[0]?.trim() ?? ""
+    if (!heading.startsWith(prefix)) {
+      return []
+    }
+    const fields = Object.fromEntries(
+      lines.slice(1).flatMap((line) => {
+        const index = line.indexOf(":")
+        return index < 0 ? [] : [[line.slice(0, index).trim(), line.slice(index + 1).trim()]]
+      })
+    )
+    return fields.files === undefined || fields.unit === undefined
+      ? []
+      : [
+          CoverageRule.make({
+            name: heading.slice(prefix.length).trim(),
+            files: fields.files,
+            unit: fields.unit
+          })
+        ]
+  })
+
+const dimensions = (body: string | undefined): ReadonlyArray<Dimension> =>
+  (body ?? "").split(/\r?\n/).flatMap((line) => {
+    const match = /^- ([^(]+)\(0\.\.(\d+)\): (.+)$/.exec(line)
+    const name = match?.[1]?.trim()
+    const maxScore = Number.parseInt(match?.[2] ?? "", 10)
+    const rubric = match?.[3]?.trim()
+    return name === undefined || rubric === undefined || !Number.isInteger(maxScore)
+      ? []
+      : [Dimension.make({ name, rubric, maxScore })]
+  })
+
+const markdownSidecars = Effect.fn("@llm4ts/flow/Pack.markdownSidecars")(function* (
+  workspace: WorkspaceShape,
+  directory: string
+): Effect.fn.Return<Readonly<Record<string, string>>, WorkspaceError> {
+  const paths = yield* workspace.discover(`${directory}/*.md`)
+  const entries: Array<readonly [string, string]> = []
+  for (const path of paths) {
+    const file = path.split("/").at(-1) ?? path
+    entries.push([file.replace(/\.md$/, ""), (yield* workspace.read(path)).trim()])
+  }
+  return Object.fromEntries(entries)
+})
+
+export const loadPack = Effect.fn("@llm4ts/flow/Pack.load")(function* (
+  workspace: WorkspaceShape,
+  directory: string
+): Effect.fn.Return<Pack, WorkspaceError | PlanParseError> {
+  const manifest = yield* parseManifest(yield* workspace.read(`${directory}/pack.md`))
+  const prompts = yield* markdownSidecars(workspace, `${directory}/prompts`)
+  const lensFiles = yield* markdownSidecars(workspace, `${directory}/reviewers`)
+  const lessons = yield* workspace.read(`${directory}/lessons.md`).pipe(
+    Effect.map((text) => text.trim()),
+    Effect.catch(() => Effect.succeed(undefined))
+  )
+  const gateValues = namedItems(section(manifest.sections, "Gates"))
+  const equivalenceValues = namedItems(section(manifest.sections, "Equivalence"))
+  const ordering: ComparisonOrdering =
+    equivalenceValues.ordering?.toLowerCase() === "unordered"
+      ? "Unordered"
+      : equivalenceValues.ordering?.toLowerCase() === "per-key"
+        ? "PerKey"
+        : "Ordered"
+  const gates = Object.fromEntries(
+    Object.entries(gateValues).map(([name, command]) => [name, command.split(/\s+/)])
+  )
+  const lenses = Object.entries(lensFiles)
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([name, text]) => parseReviewer(name, text))
+  const fields = manifest.fields
+  const pack: Pack = {
+    name: manifest.name,
+    source: fields.source ?? "",
+    scaffold: fields.scaffold,
+    sources: fields.sources,
+    programs: fields.programs,
+    specsDir: fields["specs-dir"] ?? "docs/specs",
+    featuresDir: fields["features-dir"] ?? "features",
+    gates,
+    replay: fields.replay?.split(/\s+/),
+    equivalence: ComparisonPolicy.make({
+      ordering,
+      ignore: new Set(
+        (equivalenceValues.ignore ?? "")
+          .split(",")
+          .map((item) => item.trim())
+          .filter((item) => item.length > 0)
+      )
+    }),
+    judgeDimensions: dimensions(section(manifest.sections, "Judge")),
+    coverage: rules(manifest.sections, "## Coverage: "),
+    survey: rules(manifest.sections, "## Survey: "),
+    prompts,
+    lenses,
+    lessons: lessons === undefined || lessons.length === 0 ? undefined : lessons,
+    dir: directory,
+    gate: (name) => gates[name],
+    prompt: (name) => prompts[name]
+  }
+  return pack
+})
+
+export const appendPackLesson = (
+  workspace: WorkspaceShape,
+  directory: string,
+  lesson: string
+): Effect.Effect<void, WorkspaceError> =>
+  workspace.append(`${directory}/lessons.md`, `- ${lesson.trim()}\n`)

package/src/Package.ts

@@ -0,0 +1 @@
+export const packageName = "@llm4ts/flow"

package/src/Persistence.ts

@@ -0,0 +1,198 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import { PersistenceError, PlanParseError, UnsupportedSchemaVersion } from "./FlowError.ts"
+import type { FlowError } from "./FlowError.ts"
+import { parsePlan } from "./Plan.ts"
+import type { Plan } from "./Plan.ts"
+
+export interface PlainFileStoreShape {
+  readonly read: (path: string) => Effect.Effect<string | undefined, PersistenceError>
+  readonly writeAtomic: (path: string, contents: string) => Effect.Effect<void, PersistenceError>
+  readonly append: (path: string, contents: string) => Effect.Effect<void, PersistenceError>
+  readonly remove: (path: string) => Effect.Effect<void, PersistenceError>
+  readonly hashSha256: (path: string) => Effect.Effect<string, PersistenceError>
+}
+
+export class PlainFileStore extends Context.Service<PlainFileStore, PlainFileStoreShape>()(
+  "@llm4ts/flow/PlainFileStore"
+) {}
+
+const hexDigest = (buffer: ArrayBuffer): string =>
+  Array.from(new Uint8Array(buffer))
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("")
+
+const sha256Hex = (path: string, contents: string): Effect.Effect<string, PersistenceError> =>
+  Effect.tryPromise({
+    try: async () =>
+      hexDigest(
+        await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(contents))
+      ),
+    catch: (error) =>
+      PersistenceError.make({
+        message: `failed to hash ${path}: ${error instanceof Error ? error.message : String(error)}`,
+        cause: error
+      })
+  })
+
+export interface MemoryPlainFileStore {
+  readonly store: PlainFileStoreShape
+  readonly files: Effect.Effect<Readonly<Record<string, string>>>
+  readonly replace: (files: Readonly<Record<string, string>>) => Effect.Effect<void>
+}
+
+export const makeMemoryPlainFileStore = (
+  initial: Readonly<Record<string, string>> = {}
+): Effect.Effect<MemoryPlainFileStore> =>
+  Ref.make(initial).pipe(
+    Effect.map((state) => {
+      const store: PlainFileStoreShape = {
+        read: (path) => Ref.get(state).pipe(Effect.map((files) => files[path])),
+        writeAtomic: (path, contents) =>
+          Ref.update(state, (files) => ({ ...files, [path]: contents })),
+        append: (path, contents) =>
+          Ref.update(state, (files) => ({
+            ...files,
+            [path]: `${files[path] ?? ""}${contents}`
+          })),
+        remove: (path) =>
+          Ref.update(state, (files) =>
+            Object.fromEntries(Object.entries(files).filter(([candidate]) => candidate !== path))
+          ),
+        hashSha256: (path) =>
+          Ref.get(state).pipe(
+            Effect.flatMap((files) => {
+              const contents = files[path]
+              return contents === undefined
+                ? Effect.fail(
+                    PersistenceError.make({
+                      message: `failed to hash ${path}: file does not exist`
+                    })
+                  )
+                : sha256Hex(path, contents)
+            })
+          )
+      }
+      return {
+        store,
+        files: Ref.get(state),
+        replace: (files: Readonly<Record<string, string>>) => Ref.set(state, files)
+      }
+    })
+  )
+
+export interface PlanStoreShape {
+  readonly save: (path: string, plan: Plan) => Effect.Effect<void, PersistenceError>
+  readonly load: (
+    path: string
+  ) => Effect.Effect<Plan | undefined, PersistenceError | PlanParseError>
+  readonly remove: (path: string) => Effect.Effect<void, PersistenceError>
+  readonly recoverOrCreate: <E, R>(
+    path: string,
+    create: Effect.Effect<Plan, E, R>
+  ) => Effect.Effect<Plan, E | PersistenceError | PlanParseError, R>
+}
+
+export const makePlanStore = (files: PlainFileStoreShape): PlanStoreShape => {
+  const save = (path: string, plan: Plan): Effect.Effect<void, PersistenceError> =>
+    files.writeAtomic(path, plan.render)
+
+  const load = (path: string): Effect.Effect<Plan | undefined, PersistenceError | PlanParseError> =>
+    Effect.flatMap(files.read(path), (contents) =>
+      contents === undefined
+        ? Effect.succeed(undefined)
+        : parsePlan(contents).pipe(Effect.map((plan) => plan))
+    )
+
+  const recoverOrCreate = <E, R>(
+    path: string,
+    create: Effect.Effect<Plan, E, R>
+  ): Effect.Effect<Plan, E | PersistenceError | PlanParseError, R> =>
+    Effect.flatMap(load(path), (stored) =>
+      stored === undefined ? Effect.tap(create, (plan) => save(path, plan)) : Effect.succeed(stored)
+    )
+
+  return {
+    save,
+    load,
+    remove: files.remove,
+    recoverOrCreate
+  }
+}
+
+interface VersionedEnvelope {
+  readonly schemaVersion: number
+  readonly value: unknown
+}
+
+const VersionedEnvelope = Schema.Struct({
+  schemaVersion: Schema.Int,
+  value: Schema.Unknown
+})
+
+export const saveVersioned = Effect.fn("@llm4ts/flow/Persistence.saveVersioned")(function* <
+  A,
+  E,
+  RD,
+  RE
+>(
+  files: PlainFileStoreShape,
+  path: string,
+  schemaVersion: number,
+  schema: Schema.ConstraintCodec<A, E, RD, RE>,
+  value: A
+): Effect.fn.Return<void, FlowError, RE> {
+  const encoded = yield* Schema.encodeEffect(schema)(value).pipe(
+    Effect.mapError((error) =>
+      PlanParseError.make({
+        message: `failed to encode versioned document ${path}: ${String(error)}`
+      })
+    )
+  )
+  yield* files.writeAtomic(
+    path,
+    `${JSON.stringify({ schemaVersion, value: encoded }, undefined, 2)}\n`
+  )
+})
+
+export const loadVersioned = Effect.fn("@llm4ts/flow/Persistence.loadVersioned")(function* <
+  A,
+  E,
+  RD,
+  RE
+>(
+  files: PlainFileStoreShape,
+  path: string,
+  expectedVersion: number,
+  schema: Schema.ConstraintCodec<A, E, RD, RE>
+): Effect.fn.Return<A | undefined, FlowError, RD> {
+  const contents = yield* files.read(path)
+  if (contents === undefined) {
+    return undefined
+  }
+  const envelope: VersionedEnvelope = yield* Schema.decodeUnknownEffect(
+    Schema.fromJsonString(VersionedEnvelope)
+  )(contents).pipe(
+    Effect.mapError((error) =>
+      PlanParseError.make({
+        message: `malformed versioned document ${path}: ${String(error)}`
+      })
+    )
+  )
+  if (envelope.schemaVersion !== expectedVersion) {
+    return yield* UnsupportedSchemaVersion.make({
+      path,
+      expected: expectedVersion,
+      actual: envelope.schemaVersion
+    })
+  }
+  return yield* Schema.decodeUnknownEffect(schema)(envelope.value).pipe(
+    Effect.mapError((error) =>
+      PlanParseError.make({
+        message: `invalid versioned document ${path}: ${String(error)}`
+      })
+    )
+  )
+})

package/src/Plan.ts

@@ -0,0 +1,119 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { PlanParseError } from "./FlowError.ts"
+
+export class Task extends Schema.Class<Task>("Task")({
+  title: Schema.String,
+  description: Schema.String,
+  completed: Schema.Boolean.pipe(Schema.withConstructorDefault(Effect.succeed(false)))
+}) {}
+
+export class Plan extends Schema.Class<Plan>("Plan")({
+  epicId: Schema.String,
+  tasks: Schema.Array(Task),
+  brief: Schema.optionalKey(Schema.String)
+}) {
+  get nextIncomplete(): Task | undefined {
+    return this.tasks.find((task) => !task.completed)
+  }
+
+  complete(title: string): Plan {
+    return new Plan({
+      ...this,
+      tasks: this.tasks.map((task) =>
+        task.title === title ? new Task({ ...task, completed: true }) : task
+      )
+    })
+  }
+
+  taskPrompt(task: Task): string {
+    const brief = this.brief?.trim()
+    return brief === undefined || brief.length === 0
+      ? task.description
+      : `${brief}\n\n---\n\n${task.description}`
+  }
+
+  get render(): string {
+    const blocks = this.tasks.map((task) => {
+      const box = task.completed ? "[x]" : "[ ]"
+      return `## ${box} ${task.title}\n${task.description}`.trimEnd()
+    })
+    const core = [`# Plan: ${this.epicId}`, ...blocks].join("\n\n")
+    const brief = this.brief?.trim()
+    return brief === undefined || brief.length === 0 ? core : `${core}\n\n# Brief\n\n${brief}`
+  }
+}
+
+export const stableHash = (input: string): string => {
+  let hash = 0x811c9dc5
+  for (let index = 0; index < input.length; index += 1) {
+    hash ^= input.charCodeAt(index)
+    hash = Math.imul(hash, 0x01000193)
+  }
+  return (hash >>> 0).toString(16)
+}
+
+export const defaultPlanPath = (prompt: string, directory = ".llm4ts"): string =>
+  `${directory}/plan-${stableHash(prompt)}.md`
+
+const parseTask = (chunk: string): Effect.Effect<Task, PlanParseError> => {
+  const lines = chunk.split("\n")
+  const head = lines[0] ?? ""
+  const match = /^## \[([ xX])\] (.+)$/.exec(head)
+  const marker = match?.[1]
+  const title = match?.[2]?.trim()
+  return marker === undefined || title === undefined
+    ? Effect.fail(
+        PlanParseError.make({
+          message: head.length === 0 ? "empty task chunk" : `malformed task header: ${head}`
+        })
+      )
+    : Effect.succeed(
+        Task.make({
+          title,
+          description: lines.slice(1).join("\n").trim(),
+          completed: marker.toLowerCase() === "x"
+        })
+      )
+}
+
+const splitBrief = (body: string): readonly [tasks: string, brief: string | undefined] => {
+  const lines = body.split("\n")
+  const index = lines.findIndex((line) => line.trim() === "# Brief")
+  if (index < 0) {
+    return [body.trim(), undefined]
+  }
+  const brief = lines
+    .slice(index + 1)
+    .join("\n")
+    .trim()
+  return [lines.slice(0, index).join("\n").trim(), brief.length === 0 ? undefined : brief]
+}
+
+export const parsePlan = Effect.fn("@llm4ts/flow/Plan.parse")(function* (
+  markdown: string
+): Effect.fn.Return<Plan, PlanParseError> {
+  const stripped = markdown.trim()
+  const lines = stripped.split("\n")
+  const header = lines[0] ?? ""
+  const prefix = "# Plan: "
+  if (!header.startsWith(prefix)) {
+    return yield* PlanParseError.make({
+      message: `expected a '${prefix}<epicId>' header`
+    })
+  }
+  const [taskBody, brief] = splitBrief(lines.slice(1).join("\n").trim())
+  const chunks =
+    taskBody.length === 0
+      ? []
+      : taskBody
+          .split(/^(?=## )/m)
+          .map((chunk) => chunk.trim())
+          .filter((chunk) => chunk.length > 0)
+  const tasks = yield* Effect.forEach(chunks, parseTask)
+  return Plan.make({
+    epicId: header.slice(prefix.length).trim(),
+    tasks,
+    ...(brief === undefined ? {} : { brief })
+  })
+})

package/src/PlanExecution.ts

@@ -0,0 +1,46 @@
+import * as Effect from "effect/Effect"
+import { StageCompleted, StageFailed, StageStarted, type FlowEventsShape } from "./FlowEvents.ts"
+import type { FlowError } from "./FlowError.ts"
+import type { Plan, Task } from "./Plan.ts"
+import type { PlanStoreShape } from "./Persistence.ts"
+
+const errorMessage = (error: unknown): string =>
+  error instanceof Error && error.message.length > 0 ? error.message : String(error)
+
+export const stage = <A, E, R>(
+  events: FlowEventsShape,
+  name: string,
+  effect: Effect.Effect<A, E, R>
+): Effect.Effect<A, E, R> =>
+  events.publish(StageStarted.make({ stage: name })).pipe(
+    Effect.andThen(effect),
+    Effect.tap(() => events.publish(StageCompleted.make({ stage: name }))),
+    Effect.tapError((error) =>
+      events.publish(
+        StageFailed.make({
+          stage: name,
+          message: errorMessage(error)
+        })
+      )
+    )
+  )
+
+export const implementTaskLoop = Effect.fn("@llm4ts/flow/PlanExecution.implementTaskLoop")(
+  function* <E, R>(
+    store: PlanStoreShape,
+    events: FlowEventsShape,
+    planPath: string,
+    plan: Plan,
+    perTask: (task: Task) => Effect.Effect<void, E, R>
+  ): Effect.fn.Return<Plan, E | FlowError, R> {
+    let current = plan
+    for (const task of plan.tasks) {
+      if (!task.completed) {
+        yield* stage(events, task.title, perTask(task))
+        current = current.complete(task.title)
+        yield* store.save(planPath, current)
+      }
+    }
+    return current
+  }
+)