@akterjs/effect 0.0.0-stage → 0.1.0

package/src/serve/engine/actor-residents.pure.ts

@@ -0,0 +1,500 @@
+import {
+  Array as Arr,
+  Deferred,
+  Effect,
+  Exit,
+  Fiber,
+  HashMap,
+  MutableHashMap,
+  Option,
+  Schema,
+  Scope,
+  Semaphore,
+  Stream,
+  SynchronizedRef,
+} from "effect";
+
+import { ActorUnavailableError } from "#src/contract/client/actor-unavailable.errors";
+import { ActorErasedError } from "#src/contract/actor/actor-erased.errors";
+import { InvalidRequestError } from "#src/contract/client/invalid-request.errors";
+import { FilesUnavailableError } from "#src/contract/file/files-unavailable.errors";
+import type { OpenedActor } from "#src/host/actor/actor-database.service";
+import type { QueryRequest } from "#src/host/query/query-runner.service";
+
+import { addressHolding } from "./actor-holding.pure.js";
+import type { Holding } from "./actor-holding.pure.js";
+import type { ActorAddress } from "./actor-location.service.js";
+import type { HostFailure, LiveActor, LiveReading } from "#src/serve/live/live-reads.service";
+
+type OpenFailure = ActorErasedError | ActorUnavailableError | InvalidRequestError;
+
+type Lookup<Value> = (address: ActorAddress) => Effect.Effect<Value, OpenFailure>;
+
+type Borrow<Value> = (address: ActorAddress) => Effect.Effect<Value, OpenFailure, Scope.Scope>;
+
+interface Held {
+  readonly live: LiveActor;
+  readonly opened: OpenedActor;
+}
+
+interface Cached<Resident> {
+  readonly address: ActorAddress;
+  readonly value: Resident;
+}
+
+type Using<Resident> = <Value, Failure, Needs>(
+  address: ActorAddress,
+  use: (resident: Resident) => Effect.Effect<Value, Failure, Needs>,
+) => Effect.Effect<Value, Failure | OpenFailure, Needs>;
+
+type Streaming<Resident> = <Value, Failure, Needs>(
+  address: ActorAddress,
+  use: (resident: Resident) => Stream.Stream<Value, Failure, Needs>,
+) => Stream.Stream<Value, Failure | OpenFailure, Needs>;
+
+interface Residents<Resident> {
+  readonly addresses: Effect.Effect<readonly ActorAddress[]>;
+  readonly borrow: Borrow<Resident>;
+  readonly forget: (address: ActorAddress) => Effect.Effect<void>;
+  /** Runs `effect` while no resident opens at `address`, so nothing creates its database meanwhile. */
+  readonly holding: Holding;
+  readonly use: Using<Resident>;
+  readonly stream: Streaming<Resident>;
+  readonly watch: (address: ActorAddress) => Effect.Effect<ResidentWatch<Resident>, never, Scope.Scope>;
+}
+
+interface Arrival<Resident> {
+  readonly resident: Deferred.Deferred<Resident>;
+  readonly waiting: readonly Deferred.Deferred<boolean>[];
+}
+
+interface Registered<Resident> {
+  readonly current: Option.Option<Resident>;
+  readonly ready: Deferred.Deferred<boolean>;
+  readonly resident: Deferred.Deferred<Resident>;
+}
+
+interface ResidentWatch<Resident> {
+  readonly updates: <Value, Failure, Needs>(
+    use: (resident: Resident) => Stream.Stream<Value, Failure, Needs>,
+  ) => Stream.Stream<Value, Failure, Needs>;
+  readonly current: boolean;
+  readonly ready: Effect.Effect<void>;
+}
+
+interface Cache<Resident> {
+  readonly arrivals: MutableHashMap.MutableHashMap<string, Arrival<Resident>>;
+  readonly check: (address: ActorAddress) => Effect.Effect<void, OpenFailure>;
+  readonly holding: Holding;
+  readonly residents: SynchronizedRef.SynchronizedRef<HashMap.HashMap<string, Cached<Resident>>>;
+  readonly scope: Scope.Scope;
+}
+
+type Guarded = <Value, Failure>(
+  address: ActorAddress,
+  use: (opened: OpenedActor) => Effect.Effect<Value, Failure>,
+) => Effect.Effect<Value, Failure | OpenFailure>;
+
+type Build = (address: ActorAddress, scope: Scope.Scope) => Effect.Effect<OpenedActor, OpenFailure>;
+
+interface Mark {
+  readonly changes: number;
+  readonly stamp: number;
+  readonly tables: number;
+}
+
+interface Templating {
+  readonly build: Build;
+  readonly scope: Scope.Scope;
+}
+
+interface Template {
+  readonly mark: string;
+  readonly opened: OpenedActor;
+  readonly scope: Scope.Closeable;
+}
+
+interface Blanks {
+  readonly current: (opened: OpenedActor) => LiveReading;
+  readonly template: Guarded;
+}
+
+interface Reads extends Blanks {
+  readonly stored: Lookup<boolean>;
+  readonly stream: Streaming<Held>;
+  readonly watch: Residents<Held>["watch"];
+}
+
+interface Watch<Update> {
+  readonly first: (blank: LiveReading) => Effect.Effect<Update, HostFailure>;
+  readonly updates: (live: LiveActor) => Stream.Stream<Update, HostFailure>;
+}
+
+interface LiveRoutes {
+  readonly state: (address: ActorAddress) => LiveActor["state"];
+  readonly subscribe: (call: ActorAddress & QueryRequest) => ReturnType<LiveActor["subscribe"]>;
+}
+
+interface Reading<Resident> {
+  readonly borrowExisting: Borrow<Option.Option<Resident>>;
+  readonly existing: <Value, Failure, Needs>(
+    address: ActorAddress,
+    use: (resident: Option.Option<Resident>) => Effect.Effect<Value, Failure, Needs>,
+  ) => Effect.Effect<Value, Failure | OpenFailure, Needs>;
+  readonly live: LiveRoutes;
+  readonly read: Guarded;
+}
+
+const blankVersion = 0;
+
+const bufferedInitial = 1;
+
+const onePermit = 1;
+
+const queryOnly = "PRAGMA query_only = ON";
+
+const fingerprint =
+  "SELECT total_changes() AS changes, (SELECT schema_version FROM pragma_schema_version) AS tables, (SELECT user_version FROM pragma_user_version) AS stamp";
+
+const unavailable =
+  (actor: string) =>
+  (failure: { readonly message: string }): ActorUnavailableError =>
+    ActorUnavailableError.make({ actor, message: failure.message });
+
+const unavailableUnlessErased =
+  (actor: string) =>
+  (failure: ActorErasedError | { readonly message: string }): ActorErasedError | ActorUnavailableError => {
+    if (Schema.is(ActorErasedError)(failure)) {
+      return failure;
+    }
+
+    return unavailable(actor)(failure);
+  };
+
+const residentId = ({ actor, key }: ActorAddress): string => `${actor}/${key}`;
+
+/** The identity the local engine holds an actor's files by: its name's length, its name, then its key. */
+const fileIdentity = ({ actor, key }: ActorAddress): string => `${actor.length}:${actor}${key}`;
+
+const notServed = (address: ActorAddress): InvalidRequestError =>
+  InvalidRequestError.make({ actor: address.actor, message: `${address.actor} is not served here` });
+
+const arrive = <Resident>(arrivals: Cache<Resident>["arrivals"], id: string, opened: Resident): Effect.Effect<void> =>
+  Effect.suspend(() => {
+    const waiting = MutableHashMap.get(arrivals, id);
+
+    MutableHashMap.remove(arrivals, id);
+
+    return Option.match(waiting, {
+      onNone: () => Effect.void,
+      onSome: ({ resident, waiting: watches }) =>
+        Effect.andThen(Deferred.succeed(resident, opened), Effect.forEach(watches, Deferred.await, { discard: true })),
+    });
+  });
+
+const residentOf = <Resident>(
+  { arrivals, check: cacheCheck, holding, residents, scope }: Cache<Resident>,
+  open: Lookup<Resident>,
+): Lookup<Resident> =>
+  Effect.fnUntraced(function* resident(address: ActorAddress) {
+    yield* cacheCheck(address);
+    const current = yield* SynchronizedRef.get(residents);
+    const cached = HashMap.get(current, residentId(address));
+
+    if (Option.isSome(cached)) {
+      return cached.value.value;
+    }
+
+    const opening = SynchronizedRef.modifyEffect(residents, (known) =>
+      Option.match(HashMap.get(known, residentId(address)), {
+        onNone: () =>
+          open(address).pipe(
+            Effect.tap((opened) => arrive(arrivals, residentId(address), opened)),
+            Effect.map(
+              (opened) =>
+                [
+                  opened,
+                  HashMap.set(known, residentId(address), {
+                    address: { actor: address.actor, key: address.key },
+                    value: opened,
+                  }),
+                ] as const,
+            ),
+          ),
+        onSome: (found) => Effect.succeed([found.value, known] as const),
+      }),
+    );
+
+    const started = yield* Effect.forkIn(holding(address)(opening), scope);
+
+    return yield* Fiber.join(started);
+  });
+
+const register = <Resident>(
+  { arrivals, residents }: Cache<Resident>,
+  id: string,
+): Effect.Effect<Registered<Resident>> =>
+  SynchronizedRef.modifyEffect(residents, (current) =>
+    Effect.sync(() => {
+      const pending = Option.match(MutableHashMap.get(arrivals, id), {
+        onNone: () => Deferred.makeUnsafe<Resident>(),
+        onSome: ({ resident: waiting }) => waiting,
+      });
+
+      const ready = Deferred.makeUnsafe<boolean>();
+      const found = Option.map(HashMap.get(current, id), (cached) => cached.value);
+
+      Option.match(found, {
+        onNone: () => {
+          const waiting = Option.match(MutableHashMap.get(arrivals, id), {
+            onNone: () => [],
+            onSome: (arrival) => arrival.waiting,
+          });
+
+          MutableHashMap.set(arrivals, id, { resident: pending, waiting: Arr.append(waiting, ready) });
+        },
+        onSome: (cached) => Deferred.doneUnsafe(pending, Exit.succeed(cached)),
+      });
+
+      return [{ current: found, ready, resident: pending }, current] as const;
+    }),
+  );
+
+const release = <Resident>(
+  arrivals: Cache<Resident>["arrivals"],
+  id: string,
+  registered: Registered<Resident>,
+): Effect.Effect<void> =>
+  Effect.andThen(
+    Effect.sync(() => {
+      const found = Option.filter(
+        MutableHashMap.get(arrivals, id),
+        (waiting) => waiting.resident === registered.resident,
+      );
+
+      if (Option.isNone(found)) {
+        return;
+      }
+
+      const waiting = Arr.filter(found.value.waiting, (ready) => ready !== registered.ready);
+
+      if (Arr.isReadonlyArrayEmpty(waiting)) {
+        MutableHashMap.remove(arrivals, id);
+      } else {
+        MutableHashMap.set(arrivals, id, { resident: registered.resident, waiting });
+      }
+    }),
+    Deferred.succeed(registered.ready, true),
+  ).pipe(Effect.asVoid);
+
+const watchAt =
+  <Resident>(cache: Cache<Resident>) =>
+  (address: ActorAddress): Effect.Effect<ResidentWatch<Resident>, never, Scope.Scope> =>
+    Effect.gen(function* watchArrival() {
+      const registered = yield* Effect.acquireRelease(register(cache, residentId(address)), (held) =>
+        release(cache.arrivals, residentId(address), held),
+      );
+
+      const { current, ready, resident } = registered;
+
+      return {
+        current: Option.isSome(current),
+        ready: Effect.asVoid(Deferred.succeed(ready, true)),
+        updates: (use) => Stream.unwrap(Effect.map(Deferred.await(resident), use)),
+      };
+    });
+
+const cachedResidents = <Resident>({
+  open,
+  check = () => Effect.void,
+}: {
+  readonly open: Lookup<Resident>;
+  readonly check?: Cache<Resident>["check"];
+}): Effect.Effect<Residents<Resident>, never, Scope.Scope> =>
+  Effect.gen(function* makeCachedResidents() {
+    const residents = yield* SynchronizedRef.make(HashMap.empty<string, Cached<Resident>>());
+    const scope = yield* Effect.scope;
+    const holding = yield* addressHolding(residentId);
+    const cache: Cache<Resident> = { arrivals: MutableHashMap.empty(), check, holding, residents, scope };
+    const borrow: Borrow<Resident> = residentOf(cache, open);
+
+    return {
+      addresses: Effect.map(SynchronizedRef.get(residents), (current) =>
+        Arr.map(Arr.fromIterable(HashMap.values(current)), (cached) => cached.address),
+      ),
+      borrow,
+      forget: (address) => SynchronizedRef.update(residents, (known) => HashMap.remove(known, residentId(address))),
+      holding,
+      stream: (address, use) => Stream.unwrap(Effect.map(borrow(address), use)),
+      use: (address, use) => Effect.scoped(Effect.flatMap(borrow(address), use)),
+      watch: watchAt(cache),
+    };
+  });
+
+const markOf = ({ actor, reader }: OpenedActor): Effect.Effect<string, OpenFailure> =>
+  Effect.mapBoth(reader.unsafe<Mark>(fingerprint), {
+    onFailure: unavailable(actor.name),
+    onSuccess: (rows) =>
+      Arr.join(
+        Arr.map(rows, ({ changes, stamp, tables }) => `${changes}:${tables}:${stamp}`),
+        ",",
+      ),
+  });
+
+const written = (address: ActorAddress): ActorUnavailableError =>
+  ActorUnavailableError.make({
+    actor: address.actor,
+    message: `a query wrote to ${address.actor}'s never-created template, so the template was discarded`,
+  });
+
+const builtWith =
+  (templates: MutableHashMap.MutableHashMap<string, Template>, { build, scope }: Templating) =>
+  (address: ActorAddress): Effect.Effect<Template, OpenFailure> =>
+    Option.match(MutableHashMap.get(templates, address.actor), {
+      onNone: () =>
+        Effect.gen(function* buildTemplate() {
+          const owned = yield* Scope.fork(scope);
+          const opened = yield* Effect.onError(build(address, owned), () => Scope.close(owned, Exit.void));
+          const mark = yield* markOf(opened);
+          const template = { mark, opened, scope: owned };
+
+          MutableHashMap.set(templates, address.actor, template);
+
+          return template;
+        }),
+      onSome: Effect.succeed,
+    });
+
+const blankTemplates = (templating: Templating): Effect.Effect<Guarded> =>
+  Effect.map(Semaphore.make(onePermit), (permit): Guarded => {
+    const templates = MutableHashMap.empty<string, Template>();
+    const built = builtWith(templates, templating);
+
+    return (address, use) =>
+      permit.withPermits(onePermit)(
+        Effect.gen(function* guardedRead() {
+          const template = yield* built(address);
+
+          yield* Effect.mapError(template.opened.reader.unsafe(queryOnly), unavailable(address.actor));
+
+          const exit = yield* Effect.exit(use(template.opened));
+          const mark = yield* Effect.exit(markOf(template.opened));
+
+          if (Exit.isSuccess(mark) && mark.value === template.mark) {
+            return yield* exit;
+          }
+
+          MutableHashMap.remove(templates, address.actor);
+
+          yield* Scope.close(template.scope, Exit.void);
+
+          return yield* written(address);
+        }),
+      );
+  });
+
+const attached = <Update extends { readonly version: number }>(
+  watch: ResidentWatch<Held>,
+  updates: Watch<Update>["updates"],
+): Stream.Stream<Update, HostFailure> =>
+  watch
+    .updates(({ live }) => updates(live))
+    .pipe(
+      Stream.tap(() => watch.ready),
+      Stream.filter((update) => update.version > blankVersion),
+      Stream.ensuring(watch.ready),
+    );
+
+const liveOf = <Update extends { readonly version: number }>(
+  reads: Reads,
+  address: ActorAddress,
+  { first, updates }: Watch<Update>,
+): Stream.Stream<Update, HostFailure> =>
+  Stream.unwrap(
+    Effect.flatMap(reads.stored(address), (stored) => {
+      if (stored) {
+        return Effect.succeed(reads.stream(address, ({ live }) => updates(live)));
+      }
+
+      return Effect.gen(function* watchBeforeArrival() {
+        const watch = yield* reads.watch(address);
+
+        if (watch.current) {
+          return watch.updates(({ live }) => updates(live));
+        }
+
+        const blank = yield* reads.template(address, (opened) => first(reads.current(opened)));
+
+        return Stream.concat(Stream.succeed(blank), attached(watch, updates)).pipe(
+          Stream.buffer({ capacity: bufferedInitial }),
+        );
+      });
+    }),
+  );
+
+const liveRoutes = (reads: Reads): LiveRoutes => ({
+  state: (address) => liveOf(reads, address, { first: (blank) => blank.state, updates: (live) => live.state }),
+  subscribe: (call) =>
+    liveOf(reads, call, { first: (blank) => blank.subscribe(call), updates: (live) => live.subscribe(call) }),
+});
+
+const existingWith =
+  <Resident>(resident: Borrow<Resident>, stored: Lookup<boolean>): Borrow<Option.Option<Resident>> =>
+  (address) =>
+    Effect.gen(function* existing() {
+      const found = yield* stored(address);
+
+      if (!found) {
+        return Option.none();
+      }
+
+      return yield* Effect.asSome(resident(address));
+    });
+
+interface ReadingParts<Resident> {
+  readonly blanks: Blanks;
+  readonly residents: Residents<Resident>;
+  readonly stored: Lookup<boolean>;
+}
+
+const readingOf = <Resident extends Held>({ blanks, residents, stored }: ReadingParts<Resident>): Reading<Resident> => {
+  const { current, template } = blanks;
+  const borrowExisting = existingWith(residents.borrow, stored);
+
+  const existing: Reading<Resident>["existing"] = (address, use) =>
+    Effect.scoped(Effect.flatMap(borrowExisting(address), use));
+
+  return {
+    borrowExisting,
+    existing,
+    live: liveRoutes({
+      current,
+      stored,
+      stream: residents.stream,
+      template,
+      watch: residents.watch,
+    }),
+    read: (address, use) =>
+      existing(address, Option.match({ onNone: () => template(address, use), onSome: ({ opened }) => use(opened) })),
+  };
+};
+
+type FileSession = NonNullable<OpenedActor["fileSession"]>;
+
+const fileSessionOf = ({ fileSession }: OpenedActor): Effect.Effect<FileSession, FilesUnavailableError> =>
+  Effect.fromOption(Option.fromNullishOr(fileSession)).pipe(
+    Effect.mapError(() => FilesUnavailableError.make({ message: "files aren't available on this host" })),
+  );
+
+export {
+  blankTemplates,
+  cachedResidents,
+  fileIdentity,
+  fileSessionOf,
+  notServed,
+  readingOf,
+  unavailable,
+  unavailableUnlessErased,
+};
+
+export type { OpenFailure, Reading, Residents };

package/src/serve/engine/actor-shard.pure.ts

@@ -0,0 +1,13 @@
+import { Hash, String as Str } from "effect";
+
+const shardCount = 300;
+
+const firstShard = 1;
+
+const shardDigits = 3;
+
+const shardOf = (id: string): number => Math.abs(Hash.string(id) % shardCount) + firstShard;
+
+const shardDirectory = (id: string): string => Str.padStart(shardDigits, "0")(String(shardOf(id)));
+
+export { shardCount, shardDirectory, shardOf };

package/src/serve/engine/actor-state.pure.ts

@@ -0,0 +1,45 @@
+import { Effect, Option, Stream } from "effect";
+
+import type { LiveUpdate } from "#src/contract/client/actor-rpc.contract";
+import type { CallersSchemas, HostedError, StateFields, StateSchema } from "#src/contract/actor/actor.contract";
+import { admit } from "#src/contract/caller/caller-admission.pure";
+import { startingState } from "#src/contract/hosted/hosted-request.pure";
+
+interface StatefulActor {
+  readonly callers: CallersSchemas;
+  readonly initial: Option.Option<() => StateFields>;
+  readonly name: string;
+  readonly state: StateSchema;
+}
+
+interface StateWatch<Failure> {
+  readonly actor: StatefulActor;
+  readonly principal: string;
+  readonly updates: Stream.Stream<LiveUpdate<Option.Option<string>>, Failure>;
+}
+
+const stateMember = "state";
+
+const stateUpdates = <Failure>({
+  actor,
+  principal,
+  updates,
+}: StateWatch<Failure>): Stream.Stream<LiveUpdate<StateFields>, Failure | HostedError> => {
+  const admitted = admit({
+    actor: actor.name,
+    allowed: Option.none(),
+    callers: actor.callers,
+    member: stateMember,
+    principal,
+  });
+
+  const decoded = Stream.mapEffect(updates, ({ value, version }) =>
+    Effect.map(startingState(actor, value), (state) => ({ value: state, version })),
+  );
+
+  return Stream.unwrap(Effect.as(admitted, decoded));
+};
+
+export { stateUpdates };
+
+export type { StatefulActor };

package/src/serve/engine/actors-not-hosted.errors.ts

@@ -0,0 +1,9 @@
+import { Schema } from "effect";
+
+export class ActorsNotHostedError extends Schema.TaggedError<ActorsNotHostedError, { readonly brand: unique symbol }>()(
+  "ActorsNotHostedError",
+  {
+    actors: Schema.Array(Schema.String),
+    message: Schema.String,
+  },
+) {}

package/src/serve/engine/job-runtime.layer.ts

@@ -0,0 +1,13 @@
+import { Layer } from "effect";
+import type { WorkflowEngine } from "effect/workflow";
+
+import { JobEngine } from "#src/host/job/job-engine.adapter";
+
+type JobServices = JobEngine | WorkflowEngine.WorkflowEngine;
+
+const withJobs = <Output, Failure, Needs>(
+  layer: Layer.Layer<Output, Failure, Needs>,
+): Layer.Layer<Output | JobServices, Failure, Exclude<Needs, JobServices>> =>
+  Layer.provideMerge(layer, JobEngine.layerWorkflow);
+
+export { withJobs };

package/src/serve/environment/environment-already-exists.errors.ts

@@ -0,0 +1,9 @@
+import { Schema } from "effect";
+
+import { LocalResource } from "./local-environment.schema.js";
+
+/** The name is taken: by a live snapshot of the environment, or by an environment, retired ones included. */
+export class AlreadyExistsError extends Schema.TaggedError<AlreadyExistsError, { readonly brand: unique symbol }>()(
+  "AlreadyExistsError",
+  { key: Schema.String, resource: LocalResource },
+) {}

package/src/serve/environment/environment-in-use.errors.ts

@@ -0,0 +1,17 @@
+import { Schema } from "effect";
+
+const InUseReason = Schema.Literals([
+  "environmentServing",
+  "environmentRetired",
+  "environmentIsTheDataDirectory",
+  "snapshotNotReady",
+]).annotate({
+  description:
+    "environmentServing: promote another environment first; environmentRetired: the environment named by key was deleted and keeps its name; environmentIsTheDataDirectory: the data directory's own environment can't be deleted, delete the directory instead; snapshotNotReady: the snapshot named by key is not Ready, so nothing can branch from it.",
+});
+
+/** Something the operation needs is in the way, as Akter Cloud's InUseError says. */
+export class InUseError extends Schema.TaggedError<InUseError, { readonly brand: unique symbol }>()("InUseError", {
+  key: Schema.String,
+  reason: InUseReason,
+}) {}

package/src/serve/environment/environment-not-found.errors.ts

@@ -0,0 +1,9 @@
+import { Schema } from "effect";
+
+import { LocalResource } from "./local-environment.schema.js";
+
+/** No live environment or snapshot has that name: the same error, with the same fields, Akter Cloud answers. */
+export class NotFoundError extends Schema.TaggedError<NotFoundError, { readonly brand: unique symbol }>()(
+  "NotFoundError",
+  { key: Schema.String, resource: LocalResource },
+) {}