@happy-creative/iroder 1.0.0

package/src/storage/storage.ts

@@ -0,0 +1,353 @@
+import { Log } from "../util/log"
+import path from "path"
+import { Global } from "../global"
+import { NamedError } from "@happy-creative/util/error"
+import z from "zod"
+import { AppFileSystem } from "@/filesystem"
+import { makeRuntime } from "@/effect/run-service"
+import { Effect, Exit, Layer, Option, RcMap, Schema, ServiceMap, TxReentrantLock } from "effect"
+import { Git } from "@/git"
+
+export namespace Storage {
+  const log = Log.create({ service: "storage" })
+
+  type Migration = (dir: string, fs: AppFileSystem.Interface) => Effect.Effect<void, AppFileSystem.Error>
+
+  export const NotFoundError = NamedError.create(
+    "NotFoundError",
+    z.object({
+      message: z.string(),
+    }),
+  )
+
+  export type Error = AppFileSystem.Error | InstanceType<typeof NotFoundError>
+
+  const RootFile = Schema.Struct({
+    path: Schema.optional(
+      Schema.Struct({
+        root: Schema.optional(Schema.String),
+      }),
+    ),
+  })
+
+  const SessionFile = Schema.Struct({
+    id: Schema.String,
+  })
+
+  const MessageFile = Schema.Struct({
+    id: Schema.String,
+  })
+
+  const DiffFile = Schema.Struct({
+    additions: Schema.Number,
+    deletions: Schema.Number,
+  })
+
+  const SummaryFile = Schema.Struct({
+    id: Schema.String,
+    projectID: Schema.String,
+    summary: Schema.Struct({ diffs: Schema.Array(DiffFile) }),
+  })
+
+  const decodeRoot = Schema.decodeUnknownOption(RootFile)
+  const decodeSession = Schema.decodeUnknownOption(SessionFile)
+  const decodeMessage = Schema.decodeUnknownOption(MessageFile)
+  const decodeSummary = Schema.decodeUnknownOption(SummaryFile)
+
+  export interface Interface {
+    readonly remove: (key: string[]) => Effect.Effect<void, AppFileSystem.Error>
+    readonly read: <T>(key: string[]) => Effect.Effect<T, Error>
+    readonly update: <T>(key: string[], fn: (draft: T) => void) => Effect.Effect<T, Error>
+    readonly write: <T>(key: string[], content: T) => Effect.Effect<void, AppFileSystem.Error>
+    readonly list: (prefix: string[]) => Effect.Effect<string[][], AppFileSystem.Error>
+  }
+
+  export class Service extends ServiceMap.Service<Service, Interface>()("@iroder/Storage") { }
+
+  function file(dir: string, key: string[]) {
+    return path.join(dir, ...key) + ".json"
+  }
+
+  function missing(err: unknown) {
+    if (!err || typeof err !== "object") return false
+    if ("code" in err && err.code === "ENOENT") return true
+    if ("reason" in err && err.reason && typeof err.reason === "object" && "_tag" in err.reason) {
+      return err.reason._tag === "NotFound"
+    }
+    return false
+  }
+
+  function parseMigration(text: string) {
+    const value = Number.parseInt(text, 10)
+    return Number.isNaN(value) ? 0 : value
+  }
+
+  const MIGRATIONS: Migration[] = [
+    Effect.fn("Storage.migration.1")(function* (dir: string, fs: AppFileSystem.Interface) {
+      const project = path.resolve(dir, "../project")
+      if (!(yield* fs.isDir(project))) return
+      const projectDirs = yield* fs.glob("*", {
+        cwd: project,
+        include: "all",
+      })
+      for (const projectDir of projectDirs) {
+        const full = path.join(project, projectDir)
+        if (!(yield* fs.isDir(full))) continue
+        log.info(`migrating project ${projectDir}`)
+        let projectID = projectDir
+        let worktree = "/"
+
+        if (projectID !== "global") {
+          for (const msgFile of yield* fs.glob("storage/session/message/*/*.json", {
+            cwd: full,
+            absolute: true,
+          })) {
+            const json = decodeRoot(yield* fs.readJson(msgFile), { onExcessProperty: "preserve" })
+            const root = Option.isSome(json) ? json.value.path?.root : undefined
+            if (!root) continue
+            worktree = root
+            break
+          }
+          if (!worktree) continue
+          if (!(yield* fs.isDir(worktree))) continue
+          const result = yield* Effect.promise(() =>
+            Git.run(["rev-list", "--max-parents=0", "--all"], {
+              cwd: worktree,
+            }),
+          )
+          const [id] = result
+            .text()
+            .split("\n")
+            .filter(Boolean)
+            .map((x) => x.trim())
+            .toSorted()
+          if (!id) continue
+          projectID = id
+
+          yield* fs.writeWithDirs(
+            path.join(dir, "project", projectID + ".json"),
+            JSON.stringify(
+              {
+                id,
+                vcs: "git",
+                worktree,
+                time: {
+                  created: Date.now(),
+                  initialized: Date.now(),
+                },
+              },
+              null,
+              2,
+            ),
+          )
+
+          log.info(`migrating sessions for project ${projectID}`)
+          for (const sessionFile of yield* fs.glob("storage/session/info/*.json", {
+            cwd: full,
+            absolute: true,
+          })) {
+            const dest = path.join(dir, "session", projectID, path.basename(sessionFile))
+            log.info("copying", { sessionFile, dest })
+            const session = yield* fs.readJson(sessionFile)
+            const info = decodeSession(session, { onExcessProperty: "preserve" })
+            yield* fs.writeWithDirs(dest, JSON.stringify(session, null, 2))
+            if (Option.isNone(info)) continue
+            log.info(`migrating messages for session ${info.value.id}`)
+            for (const msgFile of yield* fs.glob(`storage/session/message/${info.value.id}/*.json`, {
+              cwd: full,
+              absolute: true,
+            })) {
+              const next = path.join(dir, "message", info.value.id, path.basename(msgFile))
+              log.info("copying", {
+                msgFile,
+                dest: next,
+              })
+              const message = yield* fs.readJson(msgFile)
+              const item = decodeMessage(message, { onExcessProperty: "preserve" })
+              yield* fs.writeWithDirs(next, JSON.stringify(message, null, 2))
+              if (Option.isNone(item)) continue
+
+              log.info(`migrating parts for message ${item.value.id}`)
+              for (const partFile of yield* fs.glob(`storage/session/part/${info.value.id}/${item.value.id}/*.json`, {
+                cwd: full,
+                absolute: true,
+              })) {
+                const out = path.join(dir, "part", item.value.id, path.basename(partFile))
+                const part = yield* fs.readJson(partFile)
+                log.info("copying", {
+                  partFile,
+                  dest: out,
+                })
+                yield* fs.writeWithDirs(out, JSON.stringify(part, null, 2))
+              }
+            }
+          }
+        }
+      }
+    }),
+    Effect.fn("Storage.migration.2")(function* (dir: string, fs: AppFileSystem.Interface) {
+      for (const item of yield* fs.glob("session/*/*.json", {
+        cwd: dir,
+        absolute: true,
+      })) {
+        const raw = yield* fs.readJson(item)
+        const session = decodeSummary(raw, { onExcessProperty: "preserve" })
+        if (Option.isNone(session)) continue
+        const diffs = session.value.summary.diffs
+        yield* fs.writeWithDirs(
+          path.join(dir, "session_diff", session.value.id + ".json"),
+          JSON.stringify(diffs, null, 2),
+        )
+        yield* fs.writeWithDirs(
+          path.join(dir, "session", session.value.projectID, session.value.id + ".json"),
+          JSON.stringify(
+            {
+              ...(raw as Record<string, unknown>),
+              summary: {
+                additions: diffs.reduce((sum, x) => sum + x.additions, 0),
+                deletions: diffs.reduce((sum, x) => sum + x.deletions, 0),
+              },
+            },
+            null,
+            2,
+          ),
+        )
+      }
+    }),
+  ]
+
+  export const layer = Layer.effect(
+    Service,
+    Effect.gen(function* () {
+      const fs = yield* AppFileSystem.Service
+      const locks = yield* RcMap.make({
+        lookup: () => TxReentrantLock.make(),
+        idleTimeToLive: 0,
+      })
+      const state = yield* Effect.cached(
+        Effect.gen(function* () {
+          const dir = path.join(Global.Path.data, "storage")
+          const marker = path.join(dir, "migration")
+          const migration = yield* fs.readFileString(marker).pipe(
+            Effect.map(parseMigration),
+            Effect.catchIf(missing, () => Effect.succeed(0)),
+            Effect.orElseSucceed(() => 0),
+          )
+          for (let i = migration; i < MIGRATIONS.length; i++) {
+            log.info("running migration", { index: i })
+            const step = MIGRATIONS[i]!
+            const exit = yield* Effect.exit(step(dir, fs))
+            if (Exit.isFailure(exit)) {
+              log.error("failed to run migration", { index: i, cause: exit.cause })
+              break
+            }
+            yield* fs.writeWithDirs(marker, String(i + 1))
+          }
+          return { dir }
+        }),
+      )
+
+      const fail = (target: string): Effect.Effect<never, InstanceType<typeof NotFoundError>> =>
+        Effect.fail(new NotFoundError({ message: `Resource not found: ${target}` }))
+
+      const wrap = <A>(target: string, body: Effect.Effect<A, AppFileSystem.Error>) =>
+        body.pipe(Effect.catchIf(missing, () => fail(target)))
+
+      const writeJson = Effect.fnUntraced(function* (target: string, content: unknown) {
+        yield* fs.writeWithDirs(target, JSON.stringify(content, null, 2))
+      })
+
+      const withResolved = <A, E>(
+        key: string[],
+        fn: (target: string, rw: TxReentrantLock.TxReentrantLock) => Effect.Effect<A, E>,
+      ): Effect.Effect<A, E | AppFileSystem.Error> =>
+        Effect.scoped(
+          Effect.gen(function* () {
+            const target = file((yield* state).dir, key)
+            return yield* fn(target, yield* RcMap.get(locks, target))
+          }),
+        )
+
+      const remove: Interface["remove"] = Effect.fn("Storage.remove")(function* (key: string[]) {
+        yield* withResolved(key, (target, rw) =>
+          TxReentrantLock.withWriteLock(rw, fs.remove(target).pipe(Effect.catchIf(missing, () => Effect.void))),
+        )
+      })
+
+      const read: Interface["read"] = <T>(key: string[]) =>
+        Effect.gen(function* () {
+          const value = yield* withResolved(key, (target, rw) =>
+            TxReentrantLock.withReadLock(rw, wrap(target, fs.readJson(target))),
+          )
+          return value as T
+        })
+
+      const update: Interface["update"] = <T>(key: string[], fn: (draft: T) => void) =>
+        Effect.gen(function* () {
+          const value = yield* withResolved(key, (target, rw) =>
+            TxReentrantLock.withWriteLock(
+              rw,
+              Effect.gen(function* () {
+                const content = yield* wrap(target, fs.readJson(target))
+                fn(content as T)
+                yield* writeJson(target, content)
+                return content
+              }),
+            ),
+          )
+          return value as T
+        })
+
+      const write: Interface["write"] = (key: string[], content: unknown) =>
+        Effect.gen(function* () {
+          yield* withResolved(key, (target, rw) => TxReentrantLock.withWriteLock(rw, writeJson(target, content)))
+        })
+
+      const list: Interface["list"] = Effect.fn("Storage.list")(function* (prefix: string[]) {
+        const dir = (yield* state).dir
+        const cwd = path.join(dir, ...prefix)
+        const result = yield* fs
+          .glob("**/*", {
+            cwd,
+            include: "file",
+          })
+          .pipe(Effect.catch(() => Effect.succeed<string[]>([])))
+        return result
+          .map((x) => [...prefix, ...x.slice(0, -5).split(path.sep)])
+          .toSorted((a, b) => a.join("/").localeCompare(b.join("/")))
+      })
+
+      return Service.of({
+        remove,
+        read,
+        update,
+        write,
+        list,
+      })
+    }),
+  )
+
+  export const defaultLayer = layer.pipe(Layer.provide(AppFileSystem.defaultLayer))
+
+  const { runPromise } = makeRuntime(Service, defaultLayer)
+
+  export async function remove(key: string[]) {
+    return runPromise((svc) => svc.remove(key))
+  }
+
+  export async function read<T>(key: string[]) {
+    return runPromise((svc) => svc.read<T>(key))
+  }
+
+  export async function update<T>(key: string[], fn: (draft: T) => void) {
+    return runPromise((svc) => svc.update<T>(key, fn))
+  }
+
+  export async function write<T>(key: string[], content: T) {
+    return runPromise((svc) => svc.write(key, content))
+  }
+
+  export async function list(prefix: string[]) {
+    return runPromise((svc) => svc.list(prefix))
+  }
+}

package/src/sync/README.md

@@ -0,0 +1,179 @@
+tl;dr All of these APIs work, are properly type-checked, and are sync events are backwards compatible with `Bus`:
+
+```ts
+// The schema from `Updated` typechecks the object correctly
+SyncEvent.run(Updated, { sessionID: id, info: { title: "foo" } })
+
+// `subscribeAll` passes a generic sync event
+SyncEvent.subscribeAll((event) => {
+  // These will be type-checked correctly
+  event.id
+  event.seq
+  // This will be unknown because we are listening for all events,
+  // and this API is only used to record them
+  event.data
+})
+
+// This works, but you shouldn't publish sync event like this (should fail in the future)
+Bus.publish(Updated, { sessionID: id, info: { title: "foo" } })
+
+// Update event is fully type-checked
+Bus.subscribe(Updated, (event) => event.properties.info.title)
+
+// Update event is fully type-checked
+client.subscribe("session.updated", (evt) => evt.properties.info.title)
+```
+
+# Goal
+
+## Syncing with only one writer
+
+This system defines a basic event sourcing system for session replayability. The goal is to allow for one device to control and modify the session, and allow multiple other devices to "sync" session data. The sync works by getting a log of events to replay and replaying them locally.
+
+Because only one device is allowed to write, we don't need any kind of sophisticated distributed system clocks or causal ordering. We implement total ordering with a simple sequence id (a number) and increment it by one every time we generate an event.
+
+## Bus event integration and backwards compatibility
+
+This initial implementation aims to be fully backwards compatible. We should be able to land this without any visible changes to the user.
+
+An existing `Bus` abstraction to send events already exists. We already send events like `session.created` through the system. We should not duplicate this.
+
+The difference in event sourcing is events are sent _before_ the mutation happens, and "projectors" handle the effects and perform the mutations. This difference is subtle, and a necessary change for syncing to work.
+
+So the goal is:
+
+- Introduce a new syncing abstraction to handle event sourcing and projectors
+- Seamlessly integrate these new events into the same existing `Bus` abstraction
+- Maintain full backwards compatibility to reduce risk
+
+## My approach
+
+This directory introduces a new abstraction: `SyncEvent`. This handles all of the event sourcing.
+
+There are now "sync events" which are different than "bus events". Bus events are defined like this:
+
+```ts
+const Diff = BusEvent.define(
+  "session.diff",
+  z.object({
+    sessionID: SessionID.zod,
+    diff: Snapshot.FileDiff.array(),
+  }),
+)
+```
+
+You can do `Bus.publish(Diff, { ... })` to push these events, and `Bus.subscribe(Diff, handler)` to listen to them.
+
+Sync events are a lower-level abstraction which are similar, but also handle the requirements for recording and replaying. Defining them looks like this:
+
+```ts
+const Created = SyncEvent.define({
+  type: "session.created",
+  version: 1,
+  aggregate: "sessionID",
+  schema: z.object({
+    sessionID: SessionID.zod,
+    info: Info,
+  }),
+})
+```
+
+Not too different, except they track a version and an "aggregate" field (will explain that later).
+
+You do this to run an event, which is kind of like `Bus.publish` except that it runs through the event sourcing system:
+
+```
+SyncEvent.run(Created, { ... })
+```
+
+The data passed as the second argument is properly type-checked based on the schema defined in `Created`.
+
+Importantly, **sync events automatically re-publish as bus events**. This makes them backwards compatible, and allows the `Bus` to still be the single abstraction that the system uses to listen for individual events.
+
+**We have upgraded many of the session events to be sync events** (all of the ones that mutate the db). Sync and bus events are largely compatible. Here are the differences:
+
+### Event shape
+
+- The shape of the events are slightly different. A sync event has the `type`, `id`, `seq`, `aggregateID`, and `data` fields. A bus event has the `type` and `properties` fields. `data` and `properties` are largely the same thing. This conversion is automatically handled when the sync system re-published the event throught the bus.
+
+The reason for this is because sync events need to track more information. I chose not to copy the `properties` naming to more clearly disambiguate the event types.
+
+### Event flow
+
+There is no way to subscribe to individual sync events in `SyncEvent`. You can use `subscribeAll` to receive _all_ of the events, which is needed for clients that want to record them.
+
+To listen for individual events, use `Bus.subscribe`. You can pass in a sync event definition to it: `Bus.subscribe(Created, handler)`. This is fully supported.
+
+You should never "publish" a sync event however: `Bus.publish(Created, ...)`. I would like to force this to be a type error in the future. You should never be touching the db directly, and should not be manually handling these events.
+
+### Backwards compatibility
+
+The system install projectors in `server/projectors.js`. It calls `SyncEvent.init` to do this. It also installs a hook for dynamically converting an event at runtime (`convertEvent`).
+
+This allows you to "reshape" an event from the sync system before it's published to the bus. This should be avoided, but might be necessary for temporary backwards compat.
+
+The only time we use this is the `session.updated` event. Previously this event contained the entire session object. The sync even only contains the fields updated. We convert the event to contain to full object for backwards compatibility (but ideally we'd remove this).
+
+It's very important that types are correct when working with events. Event definitions have a `schema` which carries the defintiion of the event shape (provided by a zod schema, inferred into a TypeScript type). Examples:
+
+```ts
+// The schema from `Updated` typechecks the object correctly
+SyncEvent.run(Updated, { sessionID: id, info: { title: "foo" } })
+
+// `subscribeAll` passes a generic sync event
+SyncEvent.subscribeAll((event) => {
+  // These will be type-checked correctly
+  event.id
+  event.seq
+  // This will be unknown because we are listening for all events,
+  // and this API is only used to record them
+  event.data
+})
+
+// This works, but you shouldn't publish sync event like this (should fail in the future)
+Bus.publish(Updated, { sessionID: id, info: { title: "foo" } })
+
+// Update event is fully type-checked
+Bus.subscribe(Updated, (event) => event.properties.info.title)
+
+// Update event is fully type-checked
+client.subscribe("session.updated", (evt) => evt.properties.info.title)
+```
+
+The last two examples look similar to `SyncEvent.run`, but they were the cause of a lot of grief. Those are existing APIs that we can't break, but we are passing in the new sync event definitions to these APIs, which sometimes have a different event shape.
+
+I previously mentioned the runtime conversion of events, but we still need to the types to work! To do that, the `define` API supports an optional `busSchema` prop to give it the schema for backwards compatibility. For example this is the full definition of `Session.Update`:
+
+```ts
+const Update = SyncEvent.define({
+  type: "session.updated",
+  version: 1,
+  aggregate: "sessionID",
+  schema: z.object({
+    sessionID: SessionID.zod,
+    info: partialSchema(Info),
+  }),
+  busSchema: z.object({
+    sessionID: SessionID.zod,
+    info: Info,
+  }),
+})
+```
+
+_Important_: the conversion done in `convertEvent` is not automatically type-checked with `busSchema`. It's very important they match, but because we need this at type-checking time this needs to live here.
+
+Internally, the way this works is `busSchema` is stored on a `properties` field which is what the bus system expects. Doing this made everything with `Bus` "just work". This is why you can pass a sync event to the bus APIs.
+
+_Alternatives_
+
+These are some other paths I explored:
+
+- Providing a way to subscribe to individual sync events, and change all the instances of `Bus.subscribe` in our code to it. Then you are directly only working with sync events always.
+  - Two big problems. First, `Bus` is instance-scoped, and we'd need to make the sync event system instance-scoped too for backwards compat. If we didn't, those listeners would get calls for events they weren't expecting.
+  - Second, we can't change consumers of our SDK. So they still have to use the old events, and we might as well stick with them for consistency
+- Directly add sync event support to bus system
+  - I explored adding sync events to the bus, but due to backwards compat, it only made it more complicated (still need to support both shapes)
+- I explored a `convertSchema` function to convert the event schema at runtime so we didn't need `busSchema`
+  - Fatal flaw: we need type-checking done earlier. We can't do this at run-time. This worked for consumers of our SDK (because it gets generated TS types from the converted schema) but breaks for our internal usage of `Bus.subscribe` calls
+
+I explored many other permutations of the above solutions. What we have today I think is the best balance of backwards compatibility while opening a path forward for the new events.

package/src/sync/event.sql.ts

@@ -0,0 +1,16 @@
+import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core"
+
+export const EventSequenceTable = sqliteTable("event_sequence", {
+  aggregate_id: text().notNull().primaryKey(),
+  seq: integer().notNull(),
+})
+
+export const EventTable = sqliteTable("event", {
+  id: text().primaryKey(),
+  aggregate_id: text()
+    .notNull()
+    .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }),
+  seq: integer().notNull(),
+  type: text().notNull(),
+  data: text({ mode: "json" }).$type<Record<string, unknown>>().notNull(),
+})