@mepuka/skygent 0.2.0

package/src/services/trending-topics.ts

@@ -0,0 +1,226 @@
+/**
+ * Trending Topics Service
+ *
+ * Provides functionality to check if hashtags are trending on Bluesky.
+ * Uses the Bluesky API to fetch trending topics and caches results for 15 minutes
+ * to reduce API calls and improve performance.
+ *
+ * This service is primarily used by the Trending filter to determine if posts
+ * contain hashtags that are currently trending.
+ *
+ * @module services/trending-topics
+ * @example
+ * ```ts
+ * import { Effect } from "effect";
+ * import { TrendingTopics } from "./services/trending-topics.js";
+ * import { Hashtag } from "./domain/primitives.js";
+ *
+ * const program = Effect.gen(function* () {
+ *   const trending = yield* TrendingTopics;
+ *
+ *   // Get all trending topics
+ *   const topics = yield* trending.getTopics();
+ *   console.log("Trending topics:", topics);
+ *
+ *   // Check if a specific hashtag is trending
+ *   const isTrending = yield* trending.isTrending(Hashtag.make("effect"));
+ *   console.log("Is #effect trending?", isTrending);
+ * });
+ * ```
+ */
+
+import * as KeyValueStore from "@effect/platform/KeyValueStore";
+import {
+  Clock,
+  Context,
+  Duration,
+  Effect,
+  Layer,
+  Option,
+  Schema
+} from "effect";
+import { FilterEvalError } from "../domain/errors.js";
+import { Hashtag } from "../domain/primitives.js";
+import { BskyClient } from "./bsky-client.js";
+
+/** Cache key prefix for trending topics storage */
+const cachePrefix = "cache/trending/";
+
+/** Cache key for storing trending topics */
+const cacheKey = "topics";
+
+/** Cache TTL - trending topics are cached for 15 minutes */
+const cacheTtl = Duration.minutes(15);
+
+/**
+ * Converts an error to a FilterEvalError with the given message.
+ * @param message - Error message describing the failure context
+ * @returns A function that takes a cause and creates a FilterEvalError
+ */
+const toFilterEvalError = (message: string) => (cause: unknown) =>
+  FilterEvalError.make({ message, cause });
+
+/**
+ * Normalizes a topic string for comparison by trimming whitespace,
+ * converting to lowercase, and removing the leading '#' if present.
+ * @param topic - The topic string to normalize
+ * @returns Normalized topic string
+ * @example
+ * ```ts
+ * normalizeTopic("#Effect") // returns "effect"
+ * normalizeTopic("  BSKY  ") // returns "bsky"
+ * ```
+ */
+const normalizeTopic = (topic: string) =>
+  topic.trim().toLowerCase().replace(/^#/, "");
+
+/**
+ * Schema for cached trending topics data.
+ * Stores the list of topics and when they were fetched.
+ */
+class TrendingCacheEntry extends Schema.Class<TrendingCacheEntry>("TrendingCacheEntry")({
+  /** Array of trending topic strings */
+  topics: Schema.Array(Schema.String),
+  /** Timestamp when the cache entry was created */
+  checkedAt: Schema.DateFromString
+}) {}
+
+/**
+ * Interface for the Trending Topics service.
+ * Provides methods to fetch and check trending topics on Bluesky.
+ */
+export type TrendingTopicsService = {
+  /**
+   * Fetches the current trending topics from Bluesky.
+   * Uses a 15-minute cache to avoid excessive API calls.
+   * @returns An Effect that resolves to an array of trending topic strings
+   * @throws {FilterEvalError} If fetching topics fails
+   */
+  readonly getTopics: () => Effect.Effect<ReadonlyArray<string>, FilterEvalError>;
+
+  /**
+   * Checks if a specific hashtag is currently trending.
+   * @param tag - The hashtag to check
+   * @returns An Effect that resolves to true if the hashtag is trending, false otherwise
+   * @throws {FilterEvalError} If checking trending status fails
+   * @example
+   * ```ts
+   * const isTrending = yield* trending.isTrending(Hashtag.make("typescript"));
+   * ```
+   */
+  readonly isTrending: (tag: Hashtag) => Effect.Effect<boolean, FilterEvalError>;
+};
+
+/**
+ * Effect Context Tag for the Trending Topics service.
+ * Provides trending topic functionality with caching for Bluesky hashtags.
+ *
+ * @example
+ * ```ts
+ * // Use in an Effect program
+ * const program = Effect.gen(function* () {
+ *   const trending = yield* TrendingTopics;
+ *   const topics = yield* trending.getTopics();
+ * });
+ *
+ * // Provide the layer
+ * const runnable = program.pipe(Effect.provide(TrendingTopics.layer));
+ * ```
+ */
+export class TrendingTopics extends Context.Tag("@skygent/TrendingTopics")<
+  TrendingTopics,
+  TrendingTopicsService
+>() {
+  /**
+   * Production layer that provides the TrendingTopics service.
+   * Requires KeyValueStore and BskyClient to be provided.
+   *
+   * The implementation:
+   * - Fetches trending topics from Bluesky API
+   * - Caches results for 15 minutes
+   * - Normalizes topic names for case-insensitive comparison
+   */
+  static readonly layer = Layer.effect(
+    TrendingTopics,
+    Effect.gen(function* () {
+      const kv = yield* KeyValueStore.KeyValueStore;
+      const bsky = yield* BskyClient;
+      const store = KeyValueStore.prefix(
+        kv.forSchema(TrendingCacheEntry),
+        cachePrefix
+      );
+
+      /**
+       * Fetches trending topics from Bluesky API.
+       * Wrapped with error mapping for consistent error handling.
+       */
+      const fetchTopics = bsky.getTrendingTopics().pipe(
+        Effect.mapError(toFilterEvalError("Trending topics fetch failed"))
+      );
+
+      /**
+       * Checks if a cache entry is still fresh (within TTL).
+       * @param entry - The cached trending topics entry
+       * @param now - Current timestamp in milliseconds
+       * @returns True if the entry is still fresh
+       */
+      const isFresh = (entry: TrendingCacheEntry, now: number) =>
+        now - entry.checkedAt.getTime() < Duration.toMillis(cacheTtl);
+
+      /**
+       * Gets trending topics, using cache if available and fresh.
+       * Fetches fresh data if cache is expired or missing.
+       */
+      const getTopics = Effect.fn("TrendingTopics.getTopics")(() =>
+        Effect.gen(function* () {
+          const cached = yield* store
+            .get(cacheKey)
+            .pipe(Effect.mapError(toFilterEvalError("Trending cache read failed")));
+
+          const now = yield* Clock.currentTimeMillis;
+          if (Option.isSome(cached) && isFresh(cached.value, now)) {
+            return cached.value.topics;
+          }
+
+          const topics = yield* fetchTopics;
+          const entry = TrendingCacheEntry.make({
+            topics,
+            checkedAt: new Date(now)
+          });
+          yield* store
+            .set(cacheKey, entry)
+            .pipe(Effect.mapError(toFilterEvalError("Trending cache write failed")));
+
+          return topics;
+        })
+      );
+
+      /**
+       * Checks if a specific hashtag is in the trending topics list.
+       * Uses normalized comparison for case-insensitive matching.
+       */
+      const isTrending = Effect.fn("TrendingTopics.isTrending")((tag: Hashtag) =>
+        getTopics().pipe(
+          Effect.map((topics) =>
+            topics.includes(normalizeTopic(String(tag)))
+          )
+        )
+      );
+
+      return TrendingTopics.of({ getTopics, isTrending });
+    })
+  );
+
+  /**
+   * Test layer with mock trending topics for testing.
+   * Returns static data: ["effect", "bsky"]
+   */
+  static readonly testLayer = Layer.succeed(
+    TrendingTopics,
+    TrendingTopics.of({
+      getTopics: () => Effect.succeed(["effect", "bsky"]),
+      isTrending: (tag) =>
+        Effect.succeed(["effect", "bsky"].includes(normalizeTopic(String(tag))))
+    })
+  );
+}

package/src/services/view-checkpoint-store.ts

@@ -0,0 +1,238 @@
+/**
+ * View Checkpoint Store Service
+ *
+ * Manages checkpoints for store derivation (view) operations. Tracks the last
+ * processed event from a source store to enable incremental derivation.
+ *
+ * Unlike sync checkpoints which track data sources, view checkpoints track
+ * the processing progress when deriving one store from another, allowing
+ * incremental updates rather than full re-derivation.
+ *
+ * @module services/view-checkpoint-store
+ */
+
+import { Context, Effect, Layer, Option, Schema } from "effect";
+import { DerivationCheckpoint, FilterEvaluationMode } from "../domain/derivation.js";
+import { EventSeq, StoreName, StorePath, Timestamp } from "../domain/primitives.js";
+import { StoreIoError } from "../domain/errors.js";
+import { StoreDb } from "./store-db.js";
+import { StoreManager } from "./store-manager.js";
+
+/**
+ * Converts an error to a StoreIoError with context for the checkpoint path.
+ * @param viewName - The name of the derived store
+ * @param sourceName - The name of the source store
+ * @returns A function that creates StoreIoError from any cause
+ */
+const toStoreIoError = (viewName: StoreName, sourceName: StoreName) => (cause: unknown) => {
+  const path = Schema.decodeUnknownSync(StorePath)(
+    `stores/${viewName}/checkpoints/derivation/${sourceName}`
+  );
+  return StoreIoError.make({ path, cause });
+};
+
+const checkpointRow = Schema.Struct({
+  view_name: StoreName,
+  source_store: StoreName,
+  target_store: StoreName,
+  filter_hash: Schema.String,
+  evaluation_mode: FilterEvaluationMode,
+  last_source_event_seq: Schema.NullOr(EventSeq),
+  events_processed: Schema.NonNegativeInt,
+  events_matched: Schema.NonNegativeInt,
+  deletes_propagated: Schema.NonNegativeInt,
+  updated_at: Schema.String
+});
+
+/**
+ * Service for managing derivation (view) checkpoints.
+ *
+ * This service tracks the processing progress when deriving one store from
+ * another, storing checkpoints that record which events from the source store
+ * have been processed. This enables efficient incremental derivation.
+ */
+export class ViewCheckpointStore extends Context.Tag("@skygent/ViewCheckpointStore")<
+  ViewCheckpointStore,
+  {
+    /**
+     * Loads the derivation checkpoint for a view and its source store.
+     *
+     * @param viewName - The name of the derived store
+     * @param sourceName - The name of the source store
+     * @returns Effect resolving to Option of DerivationCheckpoint, or StoreIoError on failure
+     */
+    readonly load: (
+      viewName: StoreName,
+      sourceName: StoreName
+    ) => Effect.Effect<Option.Option<DerivationCheckpoint>, StoreIoError>;
+
+    /**
+     * Saves a derivation checkpoint.
+     *
+     * @param checkpoint - The checkpoint data to persist
+     * @returns Effect resolving to void, or StoreIoError on failure
+     */
+    readonly save: (
+      checkpoint: DerivationCheckpoint
+    ) => Effect.Effect<void, StoreIoError>;
+
+    /**
+     * Removes a derivation checkpoint.
+     *
+     * @param viewName - The name of the derived store
+     * @param sourceName - The name of the source store
+     * @returns Effect resolving to void, or StoreIoError on failure
+     */
+    readonly remove: (
+      viewName: StoreName,
+      sourceName: StoreName
+    ) => Effect.Effect<void, StoreIoError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    ViewCheckpointStore,
+    Effect.gen(function* () {
+      const storeDb = yield* StoreDb;
+      const manager = yield* StoreManager;
+
+      const resolveStore = (viewName: StoreName, sourceName: StoreName) =>
+        manager
+          .getStore(viewName)
+          .pipe(Effect.mapError(toStoreIoError(viewName, sourceName)));
+
+      const load = Effect.fn("ViewCheckpointStore.load")(
+        (viewName: StoreName, sourceName: StoreName) =>
+          resolveStore(viewName, sourceName).pipe(
+            Effect.flatMap(
+              Option.match({
+                onNone: () => Effect.succeed(Option.none<DerivationCheckpoint>()),
+                onSome: (storeRef) =>
+                  storeDb.withClient(storeRef, (client) =>
+                    Effect.gen(function* () {
+                      const rows = yield* client`SELECT
+                          view_name,
+                          source_store,
+                          target_store,
+                          filter_hash,
+                          evaluation_mode,
+                          last_source_event_seq,
+                          events_processed,
+                          events_matched,
+                          deletes_propagated,
+                          updated_at
+                        FROM derivation_checkpoints
+                        WHERE view_name = ${viewName} AND source_store = ${sourceName}`;
+                      if (rows.length === 0) {
+                        return Option.none<DerivationCheckpoint>();
+                      }
+                      const decoded = yield* Schema.decodeUnknown(
+                        Schema.Array(checkpointRow)
+                      )(rows);
+                      const row = decoded[0]!;
+                      const updatedAt = yield* Schema.decodeUnknown(Timestamp)(row.updated_at);
+                      return Option.some(
+                        DerivationCheckpoint.make({
+                          viewName: row.view_name,
+                          sourceStore: row.source_store,
+                          targetStore: row.target_store,
+                          filterHash: row.filter_hash,
+                          evaluationMode: row.evaluation_mode,
+                          lastSourceEventSeq: row.last_source_event_seq ?? undefined,
+                          eventsProcessed: row.events_processed,
+                          eventsMatched: row.events_matched,
+                          deletesPropagated: row.deletes_propagated,
+                          updatedAt
+                        })
+                      );
+                    })
+                  )
+              })
+            ),
+            Effect.mapError(toStoreIoError(viewName, sourceName))
+          )
+      );
+
+      const save = Effect.fn("ViewCheckpointStore.save")(
+        (checkpoint: DerivationCheckpoint) =>
+          resolveStore(checkpoint.viewName, checkpoint.sourceStore).pipe(
+            Effect.flatMap(
+              Option.match({
+                onNone: () =>
+                  Effect.fail(
+                    toStoreIoError(checkpoint.viewName, checkpoint.sourceStore)(
+                      new Error(`Store "${checkpoint.viewName}" not found`)
+                    )
+                  ),
+                onSome: (storeRef) =>
+                  storeDb.withClient(storeRef, (client) =>
+                    Effect.gen(function* () {
+                      const updatedAt = checkpoint.updatedAt.toISOString();
+                      yield* client`INSERT INTO derivation_checkpoints (
+                          view_name,
+                          source_store,
+                          target_store,
+                          filter_hash,
+                          evaluation_mode,
+                          last_source_event_seq,
+                          events_processed,
+                          events_matched,
+                          deletes_propagated,
+                          updated_at
+                        )
+                        VALUES (
+                          ${checkpoint.viewName},
+                          ${checkpoint.sourceStore},
+                          ${checkpoint.targetStore},
+                          ${checkpoint.filterHash},
+                          ${checkpoint.evaluationMode},
+                          ${checkpoint.lastSourceEventSeq ?? null},
+                          ${checkpoint.eventsProcessed},
+                          ${checkpoint.eventsMatched},
+                          ${checkpoint.deletesPropagated},
+                          ${updatedAt}
+                        )
+                        ON CONFLICT(view_name, source_store) DO UPDATE SET
+                          target_store = excluded.target_store,
+                          filter_hash = excluded.filter_hash,
+                          evaluation_mode = excluded.evaluation_mode,
+                          last_source_event_seq = CASE
+                            WHEN excluded.last_source_event_seq IS NULL THEN derivation_checkpoints.last_source_event_seq
+                            WHEN derivation_checkpoints.last_source_event_seq IS NULL THEN excluded.last_source_event_seq
+                            WHEN excluded.last_source_event_seq >= derivation_checkpoints.last_source_event_seq THEN excluded.last_source_event_seq
+                            ELSE derivation_checkpoints.last_source_event_seq
+                          END,
+                          events_processed = excluded.events_processed,
+                          events_matched = excluded.events_matched,
+                          deletes_propagated = excluded.deletes_propagated,
+                          updated_at = excluded.updated_at`;
+                    })
+                  )
+              })
+            ),
+            Effect.mapError(toStoreIoError(checkpoint.viewName, checkpoint.sourceStore))
+          )
+      );
+
+      const remove = Effect.fn("ViewCheckpointStore.remove")(
+        (viewName: StoreName, sourceName: StoreName) =>
+          resolveStore(viewName, sourceName).pipe(
+            Effect.flatMap(
+              Option.match({
+                onNone: () => Effect.void,
+                onSome: (storeRef) =>
+                  storeDb.withClient(storeRef, (client) =>
+                    client`DELETE FROM derivation_checkpoints
+                      WHERE view_name = ${viewName} AND source_store = ${sourceName}`.pipe(
+                      Effect.asVoid
+                    )
+                  )
+              })
+            ),
+            Effect.mapError(toStoreIoError(viewName, sourceName))
+          )
+      );
+
+      return ViewCheckpointStore.of({ load, save, remove });
+    })
+  );
+}

package/src/typeclass/chunk.ts

@@ -0,0 +1,84 @@
+import type { Applicative } from "@effect/typeclass/Applicative";
+import { getApplicative, type ConcurrencyOptions } from "@effect/typeclass/data/Effect";
+import type * as Filterable from "@effect/typeclass/Filterable";
+import type * as Traversable from "@effect/typeclass/Traversable";
+import type * as TraversableFilterable from "@effect/typeclass/TraversableFilterable";
+import { Chunk, Effect, Option } from "effect";
+import type { ChunkTypeLambda } from "effect/Chunk";
+import { dual } from "effect/Function";
+import type { Kind, TypeLambda } from "effect/HKT";
+import type { Either } from "effect/Either";
+
+export const ChunkFilterable: Filterable.Filterable<ChunkTypeLambda> = {
+  partitionMap: Chunk.partitionMap,
+  filterMap: Chunk.filterMap
+};
+
+const traverse = <F extends TypeLambda>(F: Applicative<F>) =>
+  dual(
+    2,
+    <A, R, O, E, B>(
+      self: Chunk.Chunk<A>,
+      f: (a: A) => Kind<F, R, O, E, B>
+    ): Kind<F, R, O, E, Chunk.Chunk<B>> =>
+      F.map(
+        F.productAll(Chunk.toReadonlyArray(self).map(f)),
+        Chunk.fromIterable
+      )
+  );
+
+export const ChunkTraversable: Traversable.Traversable<ChunkTypeLambda> = {
+  traverse
+};
+
+const traversePartitionMap = <F extends TypeLambda>(F: Applicative<F>) =>
+  dual(
+    2,
+    <A, R, O, E, B, C>(
+      self: Chunk.Chunk<A>,
+      f: (a: A) => Kind<F, R, O, E, Either<C, B>>
+    ): Kind<F, R, O, E, [Chunk.Chunk<B>, Chunk.Chunk<C>]> =>
+      F.map(
+        traverse(F)(self, f) as Kind<F, R, O, E, Chunk.Chunk<Either<C, B>>>,
+        (chunk) => Chunk.separate(chunk)
+      )
+  );
+
+const traverseFilterMap = <F extends TypeLambda>(F: Applicative<F>) =>
+  dual(
+    2,
+    <A, R, O, E, B>(
+      self: Chunk.Chunk<A>,
+      f: (a: A) => Kind<F, R, O, E, Option.Option<B>>
+    ): Kind<F, R, O, E, Chunk.Chunk<B>> =>
+      F.map(
+        traverse(F)(self, f) as Kind<F, R, O, E, Chunk.Chunk<Option.Option<B>>>,
+        (chunk) => Chunk.compact(chunk)
+      )
+  );
+
+export const ChunkTraversableFilterable: TraversableFilterable.TraversableFilterable<ChunkTypeLambda> = {
+  traversePartitionMap,
+  traverseFilterMap
+};
+
+export const traverseFilterEffect = <A, E, R>(
+  items: Chunk.Chunk<A>,
+  predicate: (item: A) => Effect.Effect<boolean, E, R>,
+  options?: ConcurrencyOptions
+): Effect.Effect<Chunk.Chunk<A>, E, R> =>
+  ChunkTraversableFilterable.traverseFilterMap(getApplicative(options))(
+    items,
+    (item) =>
+      Effect.map(predicate(item), (keep) =>
+        keep ? Option.some(item) : Option.none()
+      )
+  );
+
+export const filterByFlags = <A>(
+  items: Chunk.Chunk<A>,
+  flags: Chunk.Chunk<boolean>
+): Chunk.Chunk<A> =>
+  ChunkFilterable.filterMap(Chunk.zip(items, flags), (tuple) =>
+    tuple[1] ? Option.some(tuple[0]) : Option.none()
+  );