@mepuka/skygent 0.2.0

package/src/services/profile-resolver.ts

@@ -0,0 +1,223 @@
+/**
+ * Profile Resolver Service
+ *
+ * Resolves Bluesky profiles to get handles from DIDs with batching support.
+ * Uses Effect's RequestResolver for automatic batching of concurrent requests,
+ * reducing API calls by grouping multiple profile lookups together.
+ *
+ * Features:
+ * - Automatic batching of profile requests (up to 25 per batch)
+ * - Two-level caching: identity resolver cache + request-level cache
+ * - Fallback strategies: strict mode uses identity resolver, normal mode uses getProfiles API
+ *
+ * @module services/profile-resolver
+ */
+
+import {
+  Config,
+  Context,
+  Duration,
+  Effect,
+  Either,
+  Layer,
+  Option,
+  Request,
+  RequestResolver
+} from "effect";
+import { BskyError } from "../domain/errors.js";
+import { Handle } from "../domain/primitives.js";
+import { BskyClient } from "./bsky-client.js";
+import { IdentityResolver } from "./identity-resolver.js";
+
+/**
+ * Configuration for the request cache.
+ */
+type CacheConfig = {
+  readonly capacity: number;
+  readonly timeToLive: Duration.Duration;
+};
+
+/**
+ * Request class for batching profile handle lookups.
+ * Used with Effect's RequestResolver for automatic batching.
+ */
+export class ProfileHandleRequest extends Request.TaggedClass("ProfileHandle")<
+  Handle,
+  BskyError,
+  {
+    /** The DID to resolve to a handle */
+    readonly did: string;
+  }
+> {}
+
+/**
+ * Service for resolving profile handles from DIDs with batching support.
+ *
+ * This service batches concurrent handle lookups to minimize API calls.
+ * First checks the identity resolver cache, then fetches from the network
+ * in batches if needed.
+ */
+export class ProfileResolver extends Context.Tag("@skygent/ProfileResolver")<
+  ProfileResolver,
+  {
+    /**
+     * Resolves a handle for a DID, with automatic batching and caching.
+     *
+     * @param did - The DID to resolve to a handle
+     * @returns Effect resolving to Handle, or BskyError on resolution failure
+     */
+    readonly handleForDid: (did: string) => Effect.Effect<Handle, BskyError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    ProfileResolver,
+    Effect.gen(function* () {
+      const client = yield* BskyClient;
+      const identities = yield* IdentityResolver;
+
+      const batchSizeRaw = yield* Config.integer(
+        "SKYGENT_PROFILE_BATCH_SIZE"
+      ).pipe(Config.withDefault(25));
+      const batchSize = batchSizeRaw <= 0 ? 25 : Math.min(batchSizeRaw, 25);
+
+      const cacheCapacity = yield* Config.integer(
+        "SKYGENT_PROFILE_CACHE_CAPACITY"
+      ).pipe(Config.withDefault(5000));
+      const cacheTtl = yield* Config.duration("SKYGENT_PROFILE_CACHE_TTL").pipe(
+        Config.withDefault(Duration.hours(6))
+      );
+      const strict = yield* Config.boolean("SKYGENT_IDENTITY_STRICT").pipe(
+        Config.withDefault(false)
+      );
+
+      const cacheConfig =
+        cacheCapacity > 0 && Duration.toMillis(cacheTtl) > 0
+          ? Option.some<CacheConfig>({ capacity: cacheCapacity, timeToLive: cacheTtl })
+          : Option.none<CacheConfig>();
+
+      const cache = yield* Option.match(cacheConfig, {
+        onNone: () => Effect.succeed(Option.none()),
+        onSome: (config) => Request.makeCache(config).pipe(Effect.map(Option.some))
+      });
+
+      const resolveBatch = Effect.fn("ProfileResolver.resolveBatch")(
+        (requests: ReadonlyArray<ProfileHandleRequest>) =>
+          Effect.gen(function* () {
+            const dids = Array.from(new Set(requests.map((request) => request.did)));
+            if (dids.length === 0) {
+              return new Map<string, Either.Either<Handle, BskyError>>();
+            }
+
+            const cached = yield* Effect.forEach(
+              dids,
+              (did) =>
+                identities
+                  .lookupHandle(did)
+                  .pipe(Effect.either, Effect.map((result) => [did, result] as const)),
+              { concurrency: "unbounded" }
+            );
+
+            const results = new Map<string, Either.Either<Handle, BskyError>>();
+            const misses: Array<string> = [];
+
+            for (const [did, result] of cached) {
+              if (Either.isLeft(result)) {
+                results.set(did, Either.left(result.left));
+                continue;
+              }
+              if (Option.isSome(result.right)) {
+                results.set(did, Either.right(result.right.value));
+                continue;
+              }
+              misses.push(did);
+            }
+
+            if (misses.length === 0) {
+              return results;
+            }
+
+            if (strict) {
+              const resolved = yield* Effect.forEach(
+                misses,
+                (did) =>
+                  identities
+                    .resolveHandle(did)
+                    .pipe(Effect.either, Effect.map((value) => [did, value] as const)),
+                { concurrency: "unbounded" }
+              );
+              for (const [did, value] of resolved) {
+                results.set(did, value);
+              }
+              return results;
+            }
+
+            const profiles = yield* client.getProfiles(misses);
+            for (const profile of profiles) {
+              results.set(String(profile.did), Either.right(profile.handle));
+            }
+
+            yield* Effect.forEach(
+              profiles,
+              (profile) =>
+                identities.cacheProfile({
+                  did: profile.did,
+                  handle: profile.handle,
+                  source: "getProfiles",
+                  verified: false
+                }),
+              { discard: true, concurrency: "unbounded" }
+            );
+
+            return results;
+          })
+      );
+
+      const resolver = RequestResolver.makeBatched<ProfileHandleRequest, never>(
+        (requests) =>
+          resolveBatch(requests).pipe(
+            Effect.matchEffect({
+              onFailure: (error) =>
+                Effect.forEach(requests, (request) => Request.fail(request, error), {
+                  discard: true
+                }),
+              onSuccess: (profileMap) =>
+                Effect.forEach(
+                  requests,
+                  (request) => {
+                    const result = profileMap.get(request.did);
+                    if (!result) {
+                      return Request.fail(
+                        request,
+                        BskyError.make({
+                          message: `Profile not found for DID ${request.did}`,
+                          cause: request.did
+                        })
+                      );
+                    }
+                    return Either.match(result, {
+                      onLeft: (error) => Request.fail(request, error),
+                      onRight: (handle) => Request.succeed(request, handle)
+                    });
+                  },
+                  { discard: true }
+                )
+            })
+          )
+      ).pipe(RequestResolver.batchN(batchSize));
+
+      const handleForDid = Effect.fn("ProfileResolver.handleForDid")((did: string) => {
+        const effect = Effect.request(new ProfileHandleRequest({ did }), resolver);
+        return Option.match(cache, {
+          onNone: () => effect,
+          onSome: (cache) =>
+            effect.pipe(
+              Effect.withRequestCaching(true),
+              Effect.withRequestCache(cache)
+            )
+        });
+      });
+
+      return ProfileResolver.of({ handleForDid });
+    })
+  );
+}

package/src/services/resource-monitor.ts

@@ -0,0 +1,106 @@
+import { FileSystem, Path } from "@effect/platform";
+import { directorySize } from "./shared.js";
+import { Clock, Config, Context, Duration, Effect, Layer, Ref } from "effect";
+import { AppConfigService } from "./app-config.js";
+
+export type ResourceWarning =
+  | {
+      readonly _tag: "StoreSize";
+      readonly bytes: number;
+      readonly threshold: number;
+      readonly root: string;
+    }
+  | {
+      readonly _tag: "MemoryRss";
+      readonly bytes: number;
+      readonly threshold: number;
+    };
+
+export interface ResourceMonitorService {
+  readonly check: () => Effect.Effect<ReadonlyArray<ResourceWarning>>;
+}
+
+export class ResourceMonitor extends Context.Tag("@skygent/ResourceMonitor")<
+  ResourceMonitor,
+  ResourceMonitorService
+>() {
+  static readonly layer = Layer.effect(
+    ResourceMonitor,
+    Effect.gen(function* () {
+      const config = yield* AppConfigService;
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+
+      const interval = yield* Config.duration("SKYGENT_RESOURCE_CHECK_INTERVAL").pipe(
+        Config.withDefault(Duration.minutes(1))
+      );
+      const storeWarnBytes = yield* Config.integer(
+        "SKYGENT_RESOURCE_STORE_WARN_BYTES"
+      ).pipe(Config.withDefault(1_073_741_824));
+      const rssWarnBytes = yield* Config.integer(
+        "SKYGENT_RESOURCE_RSS_WARN_BYTES"
+      ).pipe(Config.withDefault(1_073_741_824));
+
+      const lastCheck = yield* Ref.make(0);
+      const intervalMs = Duration.toMillis(interval);
+
+
+      const rssUsage = () => {
+        if (
+          typeof process !== "undefined" &&
+          typeof process.memoryUsage === "function"
+        ) {
+          return process.memoryUsage().rss;
+        }
+        return 0;
+      };
+
+      const check = Effect.fn("ResourceMonitor.check")(() =>
+        Effect.gen(function* () {
+          const now = yield* Clock.currentTimeMillis;
+          const last = yield* Ref.get(lastCheck);
+          if (now - last < intervalMs) {
+            return [] as ReadonlyArray<ResourceWarning>;
+          }
+          yield* Ref.set(lastCheck, now);
+
+          const warnings: Array<ResourceWarning> = [];
+
+          if (storeWarnBytes > 0) {
+            const total = yield* directorySize(fs, path, config.storeRoot);
+            if (total >= storeWarnBytes) {
+              warnings.push({
+                _tag: "StoreSize",
+                bytes: total,
+                threshold: storeWarnBytes,
+                root: config.storeRoot
+              });
+            }
+          }
+
+          if (rssWarnBytes > 0) {
+            const rss = rssUsage();
+            if (rss >= rssWarnBytes) {
+              warnings.push({
+                _tag: "MemoryRss",
+                bytes: rss,
+                threshold: rssWarnBytes
+              });
+            }
+          }
+
+          return warnings;
+        }).pipe(Effect.orElseSucceed(() => []))
+      );
+
+      return ResourceMonitor.of({ check });
+    })
+  );
+
+  static readonly testLayer = Layer.succeed(
+    ResourceMonitor,
+    ResourceMonitor.of({
+      check: () => Effect.succeed([])
+    })
+  );
+}

package/src/services/shared.ts

@@ -0,0 +1,69 @@
+import { FileSystem, Path } from "@effect/platform";
+import { Effect, ParseResult } from "effect";
+import { ConfigError } from "../domain/errors.js";
+
+/** Extract a human-readable message from an unknown cause, falling back to the given string. */
+export const messageFromCause = (fallback: string, cause: unknown) => {
+  if (typeof cause === "object" && cause !== null && "message" in cause) {
+    const message = (cause as { readonly message?: unknown }).message;
+    if (typeof message === "string" && message.length > 0) {
+      return message;
+    }
+  }
+  return fallback;
+};
+
+/** Strip `undefined` values from a record, returning a Partial<T>. */
+export const pickDefined = <T extends Record<string, unknown>>(input: T): Partial<T> =>
+  Object.fromEntries(
+    Object.entries(input).filter(([, value]) => value !== undefined)
+  ) as Partial<T>;
+
+/** Format a Schema parse error (or arbitrary unknown) as a readable string. */
+export const formatSchemaError = (error: unknown) => {
+  if (ParseResult.isParseError(error)) {
+    return ParseResult.TreeFormatter.formatErrorSync(error);
+  }
+  return String(error);
+};
+
+/** Recursively compute the total byte-size of all files under a directory. */
+export const directorySize = (fs: FileSystem.FileSystem, path: Path.Path, root: string) =>
+  Effect.gen(function* () {
+    const exists = yield* fs.exists(root).pipe(Effect.orElseSucceed(() => false));
+    if (!exists) {
+      return 0;
+    }
+    const entries = yield* fs
+      .readDirectory(root, { recursive: true })
+      .pipe(Effect.orElseSucceed(() => []));
+    if (entries.length === 0) {
+      return 0;
+    }
+    const sizes = yield* Effect.forEach(
+      entries,
+      (entry) =>
+        fs
+          .stat(path.join(root, entry))
+          .pipe(
+            Effect.map((info) => (info.type === "File" ? Number(info.size) : 0)),
+            Effect.orElseSucceed(() => 0)
+          ),
+      { concurrency: 10 }
+    );
+    return sizes.reduce((total, size) => total + size, 0);
+  });
+
+/** Validate that a numeric config value is >= 1. */
+export const validatePositive = (name: string, value: number) => {
+  if (!Number.isFinite(value) || value < 1) {
+    return ConfigError.make({ message: `${name} must be >= 1.` });
+  }
+};
+
+/** Validate that a numeric config value is >= 0. */
+export const validateNonNegative = (name: string, value: number) => {
+  if (!Number.isFinite(value) || value < 0) {
+    return ConfigError.make({ message: `${name} must be >= 0.` });
+  }
+};

package/src/services/store-cleaner.ts

@@ -0,0 +1,43 @@
+import { Context, Effect, Layer, Option } from "effect";
+import type { StoreError } from "../domain/errors.js";
+import { StoreName } from "../domain/primitives.js";
+import { StoreDb } from "./store-db.js";
+import { StoreEventLog } from "./store-event-log.js";
+import { StoreIndex } from "./store-index.js";
+import { StoreManager } from "./store-manager.js";
+
+export class StoreCleaner extends Context.Tag("@skygent/StoreCleaner")<
+  StoreCleaner,
+  {
+    readonly deleteStore: (
+      name: StoreName
+    ) => Effect.Effect<{ readonly deleted: boolean }, StoreError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    StoreCleaner,
+    Effect.gen(function* () {
+      const manager = yield* StoreManager;
+      const index = yield* StoreIndex;
+      const eventLog = yield* StoreEventLog;
+      const storeDb = yield* StoreDb;
+
+      const deleteStore = Effect.fn("StoreCleaner.deleteStore")((name: StoreName) =>
+        Effect.gen(function* () {
+          const storeOption = yield* manager.getStore(name);
+          if (Option.isNone(storeOption)) {
+            return { deleted: false } as const;
+          }
+          const store = storeOption.value;
+          yield* eventLog.clear(store);
+          yield* index.clear(store);
+          yield* manager.deleteStore(name);
+          yield* storeDb.removeClient(name);
+          return { deleted: true } as const;
+        })
+      );
+
+      return StoreCleaner.of({ deleteStore });
+    })
+  );
+}

package/src/services/store-commit.ts

@@ -0,0 +1,168 @@
+/**
+ * Store Committer Service Module
+ *
+ * This module provides the StoreCommitter service, which handles atomic transactions
+ * for appending events to stores. It ensures data consistency by wrapping operations
+ * (upserts, conditional inserts, and deletes) in SQLite transactions.
+ *
+ * Key responsibilities:
+ * - Atomic event appending with database state changes
+ * - Deduplication support via conditional insert operations
+ * - Error handling and mapping to StoreIoError
+ */
+
+import { Context, Effect, Layer, Option } from "effect";
+import { StoreIoError } from "../domain/errors.js";
+import type { StorePath } from "../domain/primitives.js";
+import type { StoreRef } from "../domain/store.js";
+import { type EventLogEntry, PostDelete, PostUpsert } from "../domain/events.js";
+import { StoreDb } from "./store-db.js";
+import { StoreWriter } from "./store-writer.js";
+import { deletePost, insertPostIfMissing, upsertPost } from "./store-index-sql.js";
+
+const toStoreIoError = (path: StorePath) => (cause: unknown) =>
+  StoreIoError.make({ path, cause });
+
+/**
+ * Service for committing events to stores with atomic transaction guarantees.
+ *
+ * The StoreCommitter provides three core operations for persisting post events:
+ * - `appendUpsert`: Atomically upsert a post and record the event
+ * - `appendUpsertIfMissing`: Insert only if the post doesn't exist (for deduplication)
+ * - `appendDelete`: Atomically delete a post and record the event
+ *
+ * All operations are performed within SQLite transactions to maintain consistency
+ * between the post index and the event log.
+ *
+ * @example
+ * ```ts
+ * const program = Effect.gen(function* () {
+ *   const committer = yield* StoreCommitter;
+ *   const store = { root: "/data/posts" };
+ *   const event = PostUpsert.make({ ... });
+ *
+ *   const record = yield* committer.appendUpsert(store, event);
+ *   return record;
+ * });
+ * ```
+ */
+export class StoreCommitter extends Context.Tag("@skygent/StoreCommitter")<
+  StoreCommitter,
+  {
+    /**
+     * Append an upsert event to the store, updating or inserting the post.
+     *
+     * This method performs an atomic transaction that:
+     * 1. Upserts the post data into the store's index
+     * 2. Appends the event to the store's event log
+     *
+     * If either operation fails, the entire transaction is rolled back.
+     *
+     * @param store - Reference to the target store
+     * @param event - The PostUpsert event containing the post data
+     * @returns Effect that resolves to the recorded PostEventRecord
+     * @throws StoreIoError if the transaction fails
+     */
+    readonly appendUpsert: (
+      store: StoreRef,
+      event: PostUpsert
+    ) => Effect.Effect<EventLogEntry, StoreIoError>;
+
+    /**
+     * Append an upsert event only if the post doesn't already exist.
+     *
+     * This method is used for deduplication scenarios. It performs an atomic transaction that:
+     * 1. Attempts to insert the post only if it's not already present
+     * 2. If inserted, appends the event to the store's event log
+     *
+     * @param store - Reference to the target store
+     * @param event - The PostUpsert event containing the post data
+     * @returns Effect that resolves to Option.Some(record) if inserted, or Option.None() if the post already exists
+     * @throws StoreIoError if the transaction fails
+     */
+    readonly appendUpsertIfMissing: (
+      store: StoreRef,
+      event: PostUpsert
+    ) => Effect.Effect<Option.Option<EventLogEntry>, StoreIoError>;
+
+    /**
+     * Append a delete event to the store, removing the post.
+     *
+     * This method performs an atomic transaction that:
+     * 1. Deletes the post from the store's index
+     * 2. Appends the delete event to the store's event log
+     *
+     * If either operation fails, the entire transaction is rolled back.
+     *
+     * @param store - Reference to the target store
+     * @param event - The PostDelete event containing the post URI to delete
+     * @returns Effect that resolves to the recorded PostEventRecord
+     * @throws StoreIoError if the transaction fails
+     */
+    readonly appendDelete: (
+      store: StoreRef,
+      event: PostDelete
+    ) => Effect.Effect<EventLogEntry, StoreIoError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    StoreCommitter,
+    Effect.gen(function* () {
+      const storeDb = yield* StoreDb;
+      const writer = yield* StoreWriter;
+
+      const appendUpsert = Effect.fn("StoreCommitter.appendUpsert")(
+        (store: StoreRef, event: PostUpsert) =>
+          storeDb
+            .withClient(store, (client) =>
+              client.withTransaction(
+                Effect.gen(function* () {
+                  yield* upsertPost(client, event.post);
+                  return yield* writer.appendWithClient(client, event);
+                })
+              )
+            )
+            .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      const appendUpsertIfMissing = Effect.fn(
+        "StoreCommitter.appendUpsertIfMissing"
+      )((store: StoreRef, event: PostUpsert) =>
+        storeDb
+          .withClient(store, (client) =>
+            client.withTransaction(
+              Effect.gen(function* () {
+                const inserted = yield* insertPostIfMissing(client, event.post);
+                if (!inserted) {
+                  return Option.none<EventLogEntry>();
+                }
+                const entry = yield* writer.appendWithClient(client, event);
+                return Option.some(entry);
+              })
+            )
+          )
+          .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      const appendDelete = Effect.fn("StoreCommitter.appendDelete")(
+        (store: StoreRef, event: PostDelete) =>
+          storeDb
+            .withClient(store, (client) =>
+              client.withTransaction(
+                Effect.gen(function* () {
+                  yield* deletePost(client, event.uri);
+                  return yield* writer.appendWithClient(client, event);
+                })
+              )
+            )
+            .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      return StoreCommitter.of({
+        appendUpsert,
+        appendUpsertIfMissing,
+        appendDelete
+      });
+    })
+  );
+}