@smthrs/sync 0.0.0-stage → 1.0.0-rc.3

package/dist/cjs/SyncServer.js

@@ -0,0 +1,513 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var SyncServer_exports = {};
+__export(SyncServer_exports, {
+  SnapshotSource: () => SnapshotSource,
+  SyncServer: () => SyncServer,
+  defaultConcurrency: () => defaultConcurrency,
+  defaultTailIntervalMs: () => defaultTailIntervalMs,
+  layer: () => layer,
+  layerHandlers: () => layerHandlers,
+  layerNoop: () => layerNoop,
+  layerWith: () => layerWith,
+  makeLive: () => makeLive,
+  makeLiveWith: () => makeLiveWith,
+  makeNoop: () => makeNoop
+});
+module.exports = __toCommonJS(SyncServer_exports);
+var import_journal = require("@smthrs/journal");
+var Cause = __toESM(require("effect/Cause"), 1);
+var Clock = __toESM(require("effect/Clock"), 1);
+var Context = __toESM(require("effect/Context"), 1);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Layer = __toESM(require("effect/Layer"), 1);
+var Option = __toESM(require("effect/Option"), 1);
+var PubSub = __toESM(require("effect/PubSub"), 1);
+var Stream = __toESM(require("effect/Stream"), 1);
+var import_BranchProtocol = require("./BranchProtocol.js");
+var BranchShare = __toESM(require("./BranchShare.js"), 1);
+var Admission = __toESM(require("./internal/Admission.js"), 1);
+var import_CauseText = require("./internal/CauseText.js");
+var import_JournalErrorCode = require("./internal/JournalErrorCode.js");
+var import_PolicyOptions = require("./internal/PolicyOptions.js");
+var import_ProtocolVersion = require("./internal/ProtocolVersion.js");
+var SnapshotBoundary = __toESM(require("./internal/SnapshotBoundary.js"), 1);
+var RunCatalog = __toESM(require("./RunCatalog.js"), 1);
+var import_SyncError = require("./SyncError.js");
+var SyncPrincipal = __toESM(require("./SyncPrincipal.js"), 1);
+var SyncProtocol = __toESM(require("./SyncProtocol.js"), 1);
+var import_SyncRpcs = require("./SyncRpcs.js");
+class SyncServer extends Context.Service()("@smthrs/sync/SyncServer") {
+}
+class SnapshotSource extends Context.Service()("@smthrs/sync/SnapshotSource") {
+}
+const makeNoop = (overrides = {}) => SyncServer.of({
+  snapshot: () => Effect.fail(new import_SyncError.SyncError({ code: "not_found", message: "Public snapshots are unavailable" })),
+  read: Effect.fn("SyncServer.read")(() => Effect.succeed({ entries: [], cursors: [], done: true })),
+  subscribe: () => Stream.succeed({ _tag: "Closed", reason: "Sync server is unavailable" }),
+  ...overrides
+});
+const layerNoop = Layer.succeed(SyncServer, makeNoop());
+const journalFailure = (runId, generation) => (cause) => cause instanceof import_journal.Journal.JournalError && cause.code === "compacted" && cause.checkpointSeq !== void 0 ? new import_SyncError.SyncError({
+  code: "compacted",
+  message: `Run ${runId} is compacted above the requested cursor`,
+  cause: (0, import_CauseText.causeCode)(cause),
+  resync: { runId, checkpointSeq: cause.checkpointSeq, ...generation === void 0 ? {} : { generation } }
+}) : new import_SyncError.SyncError({
+  // Constant message, rendered cause. A follower may hold nothing but a
+  // branch share link, and the journal's message is the SQLite driver's:
+  // it routinely carries SQL text, table and column names, and constraint
+  // identifiers. `decodeCapability` already refuses to be a parsing
+  // oracle; this is the same rule for the read path. The CODE is a
+  // different question from the message: a journal code this boundary
+  // also declares crosses as that code, so a shut-down journal is not
+  // reported as an unexplained fault.
+  code: (0, import_JournalErrorCode.journalErrorCode)(cause),
+  message: `Journal read failed for run ${runId}`,
+  cause: (0, import_CauseText.causeCode)(cause)
+});
+const expired = new import_SyncError.SyncError({
+  code: "unauthorized",
+  message: "The capability authorizing this subscription has expired"
+});
+const cursorOf = (cursors, runId) => cursors.find((cursor) => cursor.runId === runId)?.afterSeq;
+const frameOf = (runId, generation, previous, entry) => ({
+  _tag: "Entries",
+  runId,
+  generation,
+  fromSeq: previous + 1,
+  toSeq: entry.seq,
+  entries: [entry]
+});
+const defaultConcurrency = 64;
+const defaultTailIntervalMs = 1e3;
+const tailBatchSize = 256;
+const defaults = {
+  maxFrameBytes: SyncProtocol.defaultMaxFrameBytes,
+  concurrency: defaultConcurrency,
+  tailIntervalMs: defaultTailIntervalMs
+};
+const makeWith = ({ concurrency, maxFrameBytes, tailIntervalMs }) => Effect.gen(function* () {
+  const journal = yield* import_journal.Journal.Journal;
+  const catalog = yield* RunCatalog.RunCatalog;
+  const share = yield* Effect.serviceOption(BranchShare.BranchShare);
+  const snapshots = yield* Effect.serviceOption(SnapshotSource);
+  const frameTooLarge = (bytes) => Effect.fail(
+    new import_SyncError.SyncError({
+      code: "frame_too_large",
+      message: `Encoded entries of ${bytes} bytes exceed the ${maxFrameBytes}-byte frame ceiling`
+    })
+  );
+  const guardEntryChunk = (chunk) => {
+    for (const entry of chunk) {
+      const bytes = SyncProtocol.encodedByteLength(entry);
+      if (bytes > maxFrameBytes) return frameTooLarge(bytes);
+    }
+    return Effect.void;
+  };
+  const covered = Effect.map(catalog.list, (ids) => Array.from(new Set(ids)).sort());
+  const requireWorkspace = Effect.flatMap(
+    Effect.service(SyncPrincipal.SyncPrincipal),
+    (principal) => SyncPrincipal.isWorkspace(principal) ? Effect.succeed(principal.expiresAtMs) : Effect.fail(
+      new import_SyncError.SyncError({
+        code: "unauthorized",
+        message: "Reading workspace runs requires an authenticated workspace principal"
+      })
+    )
+  );
+  const branchClaims = (branchId, capability) => Option.isNone(share) || capability === void 0 ? Effect.succeed(null) : share.value.verify(capability, { branchId, access: "read" }).pipe(
+    Effect.map((claims) => claims),
+    Effect.catch((error) => error.code === "unauthorized" ? Effect.succeed(null) : Effect.fail(error))
+  );
+  const followUntil = (runId, capability) => {
+    const branchId = (0, import_BranchProtocol.branchOfRunId)(runId);
+    return branchId === null ? Effect.succeed(Number.POSITIVE_INFINITY) : Effect.map(branchClaims(branchId, capability), (claims) => claims === null ? null : claims.expiresAtMs);
+  };
+  const runIdsFor = (scope, capability) => scope._tag === "Run" ? Effect.gen(function* () {
+    const branchId = (0, import_BranchProtocol.branchOfRunId)(scope.runId);
+    if (branchId === null) {
+      return { runIds: [scope.runId], expiresAtMs: yield* requireWorkspace };
+    }
+    const claims = yield* branchClaims(branchId, capability);
+    if (claims === null) {
+      return yield* Effect.fail(
+        new import_SyncError.SyncError({
+          code: "unauthorized",
+          message: "Reading a shared branch requires a valid share capability"
+        })
+      );
+    }
+    return { runIds: [scope.runId], expiresAtMs: claims.expiresAtMs };
+  }) : Effect.gen(function* () {
+    let expiresAtMs = yield* requireWorkspace;
+    const runIds = yield* covered;
+    const visible = [];
+    for (const runId of runIds) {
+      const branchId = (0, import_BranchProtocol.branchOfRunId)(runId);
+      if (branchId === null) {
+        visible.push(runId);
+        continue;
+      }
+      const claims = yield* branchClaims(branchId, capability);
+      if (claims === null) continue;
+      expiresAtMs = Math.min(expiresAtMs, claims.expiresAtMs);
+      visible.push(runId);
+    }
+    return { runIds: visible, expiresAtMs };
+  });
+  const requireUniqueCursors = (cursors) => {
+    const duplicate = SyncProtocol.duplicateCursorRunId(cursors);
+    return duplicate === void 0 ? Effect.void : Effect.fail(
+      new import_SyncError.SyncError({
+        code: "invalid_request",
+        message: `Cursors name run ${duplicate} more than once`
+      })
+    );
+  };
+  const generationOf = (runId, expected) => Effect.gen(function* () {
+    const current = journal.generation === void 0 ? { generation: 0, afterSeq: -1 } : yield* journal.generation(runId).pipe(Effect.mapError(journalFailure(runId)));
+    if (expected !== void 0 && current.generation !== expected) {
+      return yield* Effect.fail(
+        new import_SyncError.SyncError({
+          code: "lineage_changed",
+          message: `Run ${runId} was rewound; rebuild from the archive boundary`,
+          rewind: { runId, ...current }
+        })
+      );
+    }
+    return current.generation;
+  });
+  const readPage = (runId, after, expectedGeneration, limit) => Effect.gen(function* () {
+    const generation = yield* generationOf(runId, expectedGeneration);
+    const page = yield* journal.entries({
+      runId,
+      ...after === void 0 ? {} : { after },
+      limit
+    }).pipe(Effect.mapError(journalFailure(runId, generation)));
+    yield* generationOf(runId, generation);
+    const admitted = yield* Admission.entries(page.entries, runId, after ?? -1);
+    return { admitted, generation, hasMore: page.hasMore };
+  });
+  const snapshot = (input) => Effect.gen(function* () {
+    const request = yield* SnapshotBoundary.request(input);
+    const { expiresAtMs } = yield* runIdsFor({ _tag: "Run", runId: request.runId }, request.capability);
+    if (expiresAtMs <= (yield* Clock.currentTimeMillis)) return yield* Effect.fail(expired);
+    if (Option.isNone(snapshots)) {
+      return yield* Effect.fail(new import_SyncError.SyncError({ code: "not_found", message: "Public snapshots are unavailable" }));
+    }
+    const supplied = yield* Effect.suspend(() => snapshots.value.read({ ...request })).pipe(
+      Effect.catchCause(
+        (cause) => Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.logWarning(
+          "The public snapshot provider failed",
+          cause
+        ).pipe(Effect.andThen(Effect.fail(
+          new import_SyncError.SyncError({ code: "not_found", message: "Public snapshot is unavailable" })
+        )))
+      )
+    );
+    if (expiresAtMs <= (yield* Clock.currentTimeMillis)) return yield* Effect.fail(expired);
+    return yield* SnapshotBoundary.response(request, supplied, maxFrameBytes);
+  });
+  const read = (input) => Effect.gen(function* () {
+    const request = {
+      ...input,
+      ...yield* Admission.decode(SyncProtocol.RequestEnvelope, input, "invalid_request")
+    };
+    yield* (0, import_ProtocolVersion.requireVersion)(request.protocolVersion);
+    yield* requireUniqueCursors(request.cursors);
+    const limit = yield* (0, import_PolicyOptions.requestCount)("A read's limit", request.limit, SyncProtocol.maxReadLimit);
+    const { runIds } = yield* runIdsFor(request.scope, request.capability);
+    const entries = [];
+    const cursors = new Map(request.cursors.map((cursor) => [cursor.runId, cursor.afterSeq]));
+    const generations = new Map(request.cursors.map((cursor) => [cursor.runId, cursor.generation ?? 0]));
+    let frameBytes = 0;
+    const behind = [];
+    const serve = (runId, cap) => Effect.gen(function* () {
+      const after = cursors.get(runId);
+      const { admitted, generation, hasMore } = yield* readPage(runId, after, generations.get(runId), cap);
+      generations.set(runId, generation);
+      for (const accepted of admitted) {
+        const bytes = SyncProtocol.encodedByteLength(accepted);
+        if (bytes > maxFrameBytes) return { _tag: "oversized", bytes };
+        if (frameBytes + bytes > maxFrameBytes) return { _tag: "truncated" };
+        frameBytes += bytes;
+        entries.push(accepted);
+        cursors.set(runId, accepted.seq);
+      }
+      return hasMore ? { _tag: "more" } : { _tag: "drained" };
+    });
+    const pass = (runs, cap, more) => Effect.gen(function* () {
+      for (const runId of runs) {
+        if (entries.length >= limit) return void 0;
+        const outcome = yield* serve(runId, cap());
+        if (outcome._tag === "more") more.push(runId);
+        if (outcome._tag === "truncated" || outcome._tag === "oversized") return outcome;
+      }
+      return void 0;
+    });
+    const pending = [...runIds].sort((left, right) => (cursors.get(left) ?? -1) - (cursors.get(right) ?? -1));
+    const share2 = Math.max(1, Math.floor(limit / Math.max(runIds.length, 1)));
+    const shares = yield* pass(pending, () => Math.min(share2, limit - entries.length), behind);
+    const stop = shares ?? (yield* pass(behind, () => limit - entries.length, []));
+    if (stop?._tag === "oversized") return yield* frameTooLarge(stop.bytes);
+    return {
+      entries,
+      cursors: Array.from(cursors, ([runId, afterSeq]) => ({
+        runId,
+        afterSeq,
+        generation: generations.get(runId)
+      })),
+      // A run that reported more keeps the read unfinished even when the
+      // second pass then drained it: the page is a snapshot of a journal
+      // that is still being written, and only an untruncated pass over
+      // runs that all reported drained can promise there is nothing left.
+      done: stop === void 0 && behind.length === 0 && entries.length < limit
+    };
+  });
+  const runStream = (runId, cursors) => Stream.unwrap(Effect.gen(function* () {
+    const supplied = cursors.find((cursor) => cursor.runId === runId);
+    const after = supplied?.afterSeq;
+    const generation = yield* generationOf(runId, supplied === void 0 ? void 0 : supplied.generation ?? 0);
+    let admitted = after ?? -1;
+    const entries = journal.stream({ runId, ...after === void 0 ? {} : { afterSequence: after } }).pipe(
+      Stream.mapError(journalFailure(runId, generation)),
+      Stream.chunks,
+      Stream.mapEffect(
+        (chunk) => Admission.entries(chunk, runId, admitted).pipe(
+          Effect.tap(() => generationOf(runId, generation)),
+          Effect.tap(guardEntryChunk),
+          Effect.map((captured) => {
+            for (const entry of captured) admitted = entry.seq;
+            return captured;
+          })
+        )
+      ),
+      Stream.flattenArray,
+      Stream.mapAccum(
+        () => after === void 0 ? -1 : after,
+        (previous, entry) => [entry.seq, [frameOf(runId, generation, previous, entry)]]
+      )
+    );
+    if (journal.generation === void 0) return entries;
+    const guard = Stream.drain(Stream.fromEffectRepeat(
+      generationOf(runId, generation).pipe(Effect.delay(tailIntervalMs))
+    ));
+    return Stream.merge(entries, guard, { haltStrategy: "either" });
+  }));
+  const workspaceStream = (covering, openedUntil, request) => Stream.unwrap(Effect.gen(function* () {
+    const served = /* @__PURE__ */ new Map();
+    const generations = new Map(request.cursors.map((cursor) => [cursor.runId, cursor.generation ?? 0]));
+    const excluded = /* @__PURE__ */ new Set();
+    let visible = new Set(covering);
+    let deadline = openedUntil;
+    let dirty = /* @__PURE__ */ new Set();
+    let announced = false;
+    let reconciledAtMs = -Infinity;
+    const lowered = yield* PubSub.sliding(1);
+    const loweredSignal = yield* PubSub.subscribe(lowered);
+    const cover = (runId) => {
+      served.set(runId, cursorOf(request.cursors, runId));
+    };
+    for (const runId of covering) cover(runId);
+    const wake = yield* PubSub.sliding(1);
+    const woken = yield* PubSub.subscribe(wake);
+    const reconcile = Effect.gen(function* () {
+      const present = new Set(yield* catalog.list);
+      for (const runId of excluded) {
+        if (!present.has(runId)) excluded.delete(runId);
+      }
+      const next = [];
+      for (const runId of present) {
+        if (excluded.has(runId)) continue;
+        if (!served.has(runId)) {
+          const until = yield* followUntil(runId, request.capability);
+          if (until === null) {
+            excluded.add(runId);
+            continue;
+          }
+          if (until < deadline) {
+            deadline = until;
+            yield* PubSub.publish(lowered, void 0);
+          }
+          cover(runId);
+        }
+        next.push(runId);
+      }
+      reconciledAtMs = yield* Clock.currentTimeMillis;
+      visible = new Set(next);
+    });
+    const tail = (runId) => Stream.unwrap(Effect.gen(function* () {
+      const after = served.get(runId);
+      const { admitted, generation, hasMore } = yield* readPage(
+        runId,
+        after,
+        generations.get(runId),
+        tailBatchSize
+      );
+      generations.set(runId, generation);
+      yield* guardEntryChunk(admitted);
+      const frames = [];
+      let previous = after === void 0 ? -1 : after;
+      for (const accepted of admitted) {
+        frames.push(frameOf(runId, generation, previous, accepted));
+        previous = accepted.seq;
+      }
+      const last = admitted.at(-1);
+      if (last === void 0) return Stream.empty;
+      served.set(runId, last.seq);
+      if (hasMore) {
+        dirty.add(runId);
+        yield* PubSub.publish(wake, void 0);
+      }
+      return Stream.fromIterable(frames);
+    }));
+    const fullRound = Stream.unwrap(Effect.map(
+      reconcile,
+      () => {
+        dirty = /* @__PURE__ */ new Set();
+        announced = false;
+        return Stream.flatMap(Stream.fromIterable(visible), tail, { concurrency });
+      }
+    ));
+    const dirtyRound = Stream.suspend(() => {
+      const runs = dirty;
+      dirty = /* @__PURE__ */ new Set();
+      return Stream.flatMap(
+        Stream.fromIterable(Array.from(runs).filter((runId) => visible.has(runId))),
+        tail,
+        { concurrency }
+      );
+    });
+    const roundFor = (tick) => Effect.map(
+      Clock.currentTimeMillis,
+      (nowMs) => tick === "interval" || announced || nowMs - reconciledAtMs >= tailIntervalMs ? fullRound : dirtyRound
+    );
+    const announcements = Stream.drain(Stream.tap(
+      catalog.changes,
+      () => Effect.suspend(() => {
+        announced = true;
+        return PubSub.publish(wake, void 0);
+      })
+    ));
+    const commits = yield* journal.changes;
+    const appends = Stream.fromSubscription(commits).pipe(
+      Stream.filter((entry) => served.has(entry.runId)),
+      Stream.tap(
+        (entry) => Effect.suspend(() => {
+          dirty.add(entry.runId);
+          return PubSub.publish(wake, void 0);
+        })
+      ),
+      Stream.drain
+    );
+    const ticks = Stream.fromEffectRepeat(
+      Effect.raceFirst(
+        Effect.as(PubSub.take(woken), "wake"),
+        Effect.as(Effect.sleep(tailIntervalMs), "interval")
+      )
+    );
+    const untilDeadline = Effect.gen(function* () {
+      while (true) {
+        const nowMs = yield* Clock.currentTimeMillis;
+        if (nowMs >= deadline) {
+          return yield* Effect.fail(expired);
+        }
+        yield* Number.isFinite(deadline) ? Effect.raceFirst(Effect.sleep(deadline - nowMs), PubSub.take(loweredSignal)) : PubSub.take(loweredSignal);
+      }
+    });
+    return Stream.interruptWhen(
+      Stream.merge(
+        Stream.merge(
+          Stream.concat(
+            fullRound,
+            Stream.flatMap(ticks, (tick) => Stream.unwrap(roundFor(tick)))
+          ),
+          announcements
+        ),
+        appends
+      ),
+      untilDeadline
+    );
+  }));
+  const untilExpiry = (expiresAtMs, stream) => Number.isFinite(expiresAtMs) ? Stream.interruptWhen(
+    stream,
+    Effect.flatMap(
+      Clock.currentTimeMillis,
+      (nowMs) => Effect.andThen(Effect.sleep(Math.max(expiresAtMs - nowMs, 0)), Effect.fail(expired))
+    )
+  ) : stream;
+  const subscribe = (input) => Stream.unwrap(
+    Effect.gen(function* () {
+      const request = {
+        ...input,
+        ...yield* Admission.decode(SyncProtocol.RequestEnvelope, input, "invalid_request")
+      };
+      yield* (0, import_ProtocolVersion.requireVersion)(request.protocolVersion);
+      yield* requireUniqueCursors(request.cursors);
+      const credit = yield* (0, import_PolicyOptions.requestCount)(
+        "A subscription's credit",
+        request.credit,
+        SyncProtocol.maxSubscribeCredit
+      );
+      const { expiresAtMs, runIds } = yield* runIdsFor(request.scope, request.capability);
+      const frames = request.scope._tag === "Run" ? untilExpiry(expiresAtMs, runStream(request.scope.runId, request.cursors)) : workspaceStream(runIds, expiresAtMs, request);
+      return Stream.take(frames, credit);
+    })
+  );
+  return SyncServer.of({ snapshot, read, subscribe });
+});
+const makeLive = makeWith(defaults);
+const makeLiveWith = (options = {}) => Effect.flatMap(
+  Effect.all({
+    maxFrameBytes: (0, import_PolicyOptions.positiveInt)("SyncServer.Options.maxFrameBytes", options.maxFrameBytes, defaults.maxFrameBytes),
+    concurrency: (0, import_PolicyOptions.positiveInt)("SyncServer.Options.concurrency", options.concurrency, defaults.concurrency),
+    tailIntervalMs: (0, import_PolicyOptions.positiveInt)(
+      "SyncServer.Options.tailIntervalMs",
+      options.tailIntervalMs,
+      defaults.tailIntervalMs
+    )
+  }),
+  makeWith
+);
+const layer = Layer.effect(
+  SyncServer,
+  makeLive
+);
+const layerWith = (options) => Layer.effect(SyncServer, makeLiveWith(options));
+const layerHandlers = import_SyncRpcs.SyncRpcs.toLayer(
+  Effect.gen(function* () {
+    const sync = yield* SyncServer;
+    return import_SyncRpcs.SyncRpcs.of({
+      "Sync.Snapshot": (request) => sync.snapshot(request),
+      "Sync.Read": (request) => sync.read(request),
+      "Sync.Subscribe": (request) => sync.subscribe(request)
+    });
+  })
+);
+//# sourceMappingURL=SyncServer.js.map