@mepuka/skygent 0.2.0

package/src/services/lineage-store.ts

@@ -0,0 +1,89 @@
+/**
+ * Lineage Store Service
+ *
+ * Tracks store derivation lineage, recording parent-child relationships between
+ * stores. This enables understanding which stores are derived from others and
+ * managing derivation dependencies.
+ *
+ * Lineage information is used to:
+ * - Track which source stores a derived store depends on
+ * - Validate that derivation sources haven't changed unexpectedly
+ * - Rebuild derivation chains when needed
+ *
+ * @module services/lineage-store
+ */
+
+import * as KeyValueStore from "@effect/platform/KeyValueStore";
+import { Context, Effect, Layer, Option, Schema } from "effect";
+import { StoreLineage } from "../domain/derivation.js";
+import { StoreName, StorePath } from "../domain/primitives.js";
+import { StoreIoError } from "../domain/errors.js";
+
+/**
+ * Generates the storage key for a store's lineage information.
+ * @param storeName - The name of the store
+ * @returns The storage key string
+ */
+const lineageKey = (storeName: StoreName) => `stores/${storeName}/lineage`;
+
+/**
+ * Converts an error to a StoreIoError with context for the lineage path.
+ * @param storeName - The name of the store
+ * @returns A function that creates StoreIoError from any cause
+ */
+const toStoreIoError = (storeName: StoreName) => (cause: unknown) => {
+  const path = Schema.decodeUnknownSync(StorePath)(`stores/${storeName}/lineage`);
+  return StoreIoError.make({ path, cause });
+};
+
+/**
+ * Service for managing store derivation lineage.
+ *
+ * This service tracks parent-child relationships between stores, recording
+ * which source stores were used to derive a given store. This enables
+ * dependency tracking and validation of derivation chains.
+ */
+export class LineageStore extends Context.Tag("@skygent/LineageStore")<
+  LineageStore,
+  {
+    /**
+     * Retrieves the lineage information for a store.
+     *
+     * @param storeName - The name of the store to get lineage for
+     * @returns Effect resolving to Option of StoreLineage, or StoreIoError on failure
+     */
+    readonly get: (
+      storeName: StoreName
+    ) => Effect.Effect<Option.Option<StoreLineage>, StoreIoError>;
+
+    /**
+     * Saves lineage information for a store.
+     *
+     * @param lineage - The lineage data to persist
+     * @returns Effect resolving to void, or StoreIoError on failure
+     */
+    readonly save: (lineage: StoreLineage) => Effect.Effect<void, StoreIoError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    LineageStore,
+    Effect.gen(function* () {
+      const kv = yield* KeyValueStore.KeyValueStore;
+      const lineages = kv.forSchema(StoreLineage);
+
+      const get = Effect.fn("LineageStore.get")((storeName: StoreName) =>
+        lineages
+          .get(lineageKey(storeName))
+          .pipe(Effect.mapError(toStoreIoError(storeName)))
+      );
+
+      const save = Effect.fn("LineageStore.save")((lineage: StoreLineage) =>
+        lineages
+          .set(lineageKey(lineage.storeName), lineage)
+          .pipe(Effect.mapError(toStoreIoError(lineage.storeName)))
+      );
+
+      return LineageStore.of({ get, save });
+    })
+  );
+}

package/src/services/link-validator.ts

@@ -0,0 +1,244 @@
+/**
+ * Link Validator Service
+ *
+ * Validates external URLs found in Bluesky posts by performing HTTP HEAD requests.
+ * Used by the HasValidLinks filter to check if post links are accessible.
+ *
+ * **Validation Process:**
+ * 1. Checks URL format (must be http:// or https://)
+ * 2. Attempts HEAD request
+ * 3. Falls back to GET if HEAD returns 405 (Method Not Allowed) or 501 (Not Implemented)
+ * 4. Returns true for 2xx-3xx status codes
+ *
+ * **Caching:**
+ * Results are cached for 6 hours (configurable via cacheTtl) to avoid
+ * repeated requests to the same URLs. Cache entries include:
+ * - URL (encoded as cache key)
+ * - ok: boolean (whether link is valid)
+ * - status: number (HTTP status code, optional)
+ * - checkedAt: Date (when validation occurred)
+ *
+ * **Performance:**
+ * Cached results are used if entry is fresh (within TTL).
+ * hasValidLink returns true on first valid URL in the array (short-circuits).
+ *
+ * @module services/link-validator
+ *
+ * @example
+ * ```typescript
+ * import { LinkValidator } from "./services/link-validator.js";
+ * import { Effect } from "effect";
+ *
+ * const program = Effect.gen(function* () {
+ *   const validator = yield* LinkValidator;
+ *
+ *   // Validate a single link
+ *   const isValid = yield* validator.isValid("https://example.com");
+ *   console.log(`Link valid: ${isValid}`);
+ *
+ *   // Check if any link in array is valid
+ *   const hasValid = yield* validator.hasValidLink([
+ *     "https://example.com",
+ *     "https://invalid-url.xyz"
+ *   ]);
+ *   console.log(`Has valid link: ${hasValid}`); // true if at least one valid
+ * }).pipe(Effect.provide(LinkValidator.layer));
+ * ```
+ */
+
+import { HttpClient } from "@effect/platform";
+import * as KeyValueStore from "@effect/platform/KeyValueStore";
+import { Clock, Context, Duration, Effect, Layer, Option, Schema } from "effect";
+import { FilterEvalError } from "../domain/errors.js";
+
+const cachePrefix = "cache/links/";
+const cacheTtl = Duration.hours(6);
+
+const toFilterEvalError = (message: string) => (cause: unknown) =>
+  FilterEvalError.make({ message, cause });
+
+const isHttpUrl = (value: string) => {
+  try {
+    const url = new URL(value);
+    return url.protocol === "http:" || url.protocol === "https:";
+  } catch {
+    return false;
+  }
+};
+
+const isValidStatus = (status: number) => status >= 200 && status < 400;
+
+const cacheKey = (url: string) => encodeURIComponent(url);
+
+class LinkCacheEntry extends Schema.Class<LinkCacheEntry>("LinkCacheEntry")({
+  url: Schema.String,
+  ok: Schema.Boolean,
+  status: Schema.optional(Schema.Number),
+  checkedAt: Schema.DateFromString
+}) {}
+
+/**
+ * Interface for the link validator service.
+ * Provides URL validation with caching support.
+ */
+export type LinkValidatorService = {
+  /**
+   * Validates a single URL by checking HTTP accessibility.
+   * Returns cached result if available and fresh (within 6 hour TTL).
+   *
+   * @param url - The URL to validate
+   * @returns Effect resolving to true if link is accessible (2xx-3xx status), false otherwise
+   * @throws {FilterEvalError} When HTTP request fails unexpectedly or cache operations fail
+   */
+  readonly isValid: (url: string) => Effect.Effect<boolean, FilterEvalError>;
+
+  /**
+   * Checks if any URL in the array is valid.
+   * Short-circuits on first valid URL for performance.
+   * Non-HTTP URLs are skipped (not considered valid).
+   *
+   * @param urls - Array of URLs to check
+   * @returns Effect resolving to true if at least one URL is valid
+   * @throws {FilterEvalError} When validation fails
+   */
+  readonly hasValidLink: (
+    urls: ReadonlyArray<string>
+  ) => Effect.Effect<boolean, FilterEvalError>;
+};
+
+/**
+ * Context tag and Layer implementation for the link validator service.
+ * Performs HTTP validation with caching via KeyValueStore.
+ *
+ * **HTTP Behavior:**
+ * - Uses HEAD requests for efficiency
+ * - Falls back to GET for servers that don't support HEAD (405/501)
+ * - Considers 2xx-3xx status codes as valid
+ * - Non-HTTP URLs (ftp://, file://, etc.) are considered invalid
+ *
+ * **Caching:**
+ * Uses KeyValueStore with "cache/links/" prefix.
+ * Cache entries expire after 6 hours.
+ *
+ * **Dependencies:**
+ * - KeyValueStore.KeyValueStore: For caching
+ * - HttpClient.HttpClient: For making requests
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const isValid = yield* validator.isValid("https://example.com");
+ *
+ * // Check multiple links (stops at first valid)
+ * const hasAnyValid = yield* validator.hasValidLink([
+ *   "https://broken.example",
+ *   "https://working.example"
+ * ]);
+ *
+ * // Testing with mock responses
+ * const testLayer = Layer.succeed(
+ *   LinkValidator,
+ *   LinkValidator.of({
+ *     isValid: (url) => Effect.succeed(url.includes("ok")),
+ *     hasValidLink: (urls) => Effect.succeed(urls.some(u => u.includes("ok")))
+ *   })
+ * );
+ * ```
+ */
+export class LinkValidator extends Context.Tag("@skygent/LinkValidator")<
+  LinkValidator,
+  LinkValidatorService
+>() {
+  /**
+   * Layer that creates the link validator service.
+   * Requires KeyValueStore and HttpClient services.
+   */
+  static readonly layer = Layer.effect(
+    LinkValidator,
+    Effect.gen(function* () {
+      const kv = yield* KeyValueStore.KeyValueStore;
+      const http = yield* HttpClient.HttpClient;
+      const store = KeyValueStore.prefix(kv.forSchema(LinkCacheEntry), cachePrefix);
+
+      const isFresh = (entry: LinkCacheEntry, now: number) =>
+        now - entry.checkedAt.getTime() < Duration.toMillis(cacheTtl);
+
+      const fetchStatus = (url: string) =>
+        http.head(url).pipe(
+          Effect.map((response) => response.status),
+          Effect.flatMap((status) =>
+            status === 405 || status === 501
+              ? http.get(url).pipe(Effect.map((response) => response.status))
+              : Effect.succeed(status)
+          ),
+          Effect.mapError(toFilterEvalError("Link validation failed"))
+        );
+
+      const isValid = Effect.fn("LinkValidator.isValid")((url: string) =>
+        Effect.gen(function* () {
+          if (!isHttpUrl(url)) {
+            return false;
+          }
+
+          const now = yield* Clock.currentTimeMillis;
+          const cached = yield* store
+            .get(cacheKey(url))
+            .pipe(Effect.mapError(toFilterEvalError("Link cache read failed")));
+
+          if (Option.isSome(cached) && isFresh(cached.value, now)) {
+            return cached.value.ok;
+          }
+
+          const status = yield* fetchStatus(url);
+          const ok = isValidStatus(status);
+          const entry = LinkCacheEntry.make({
+            url,
+            ok,
+            status,
+            checkedAt: new Date(now)
+          });
+
+          yield* store
+            .set(cacheKey(url), entry)
+            .pipe(Effect.mapError(toFilterEvalError("Link cache write failed")));
+
+          return ok;
+        })
+      );
+
+      const hasValidLink = Effect.fn("LinkValidator.hasValidLink")(
+        (urls: ReadonlyArray<string>) =>
+          Effect.findFirst(urls, (url) => isValid(url)).pipe(
+            Effect.map(Option.isSome)
+          )
+      );
+
+      return LinkValidator.of({ isValid, hasValidLink });
+    })
+  );
+
+  /**
+   * Test layer that provides a mock link validator.
+   * URLs containing "ok" are considered valid.
+   * Useful for testing without making actual HTTP requests.
+   *
+   * @example
+   * ```typescript
+   * // Mock validator that considers URLs with "ok" as valid
+   * const testProgram = program.pipe(
+   *   Effect.provide(LinkValidator.testLayer)
+   * );
+   *
+   * // "https://example.com/ok" -> true
+   * // "https://broken.example" -> false
+   * ```
+   */
+  static readonly testLayer = Layer.succeed(
+    LinkValidator,
+    LinkValidator.of({
+      isValid: (url) => Effect.succeed(url.includes("ok")),
+      hasValidLink: (urls) =>
+        Effect.succeed(urls.some((url) => url.includes("ok")))
+    })
+  );
+}

package/src/services/output-manager.ts

@@ -0,0 +1,274 @@
+import { FileSystem, Path } from "@effect/platform";
+import { Chunk, Clock, Context, Effect, Layer, Option, Ref, Schema, Stream } from "effect";
+import { StoreQuery } from "../domain/events.js";
+import type { Post } from "../domain/post.js";
+import { Post as PostSchema } from "../domain/post.js";
+import type { FilterSpec, StoreRef } from "../domain/store.js";
+import { FilterCompiler } from "./filter-compiler.js";
+import { FilterRuntime } from "./filter-runtime.js";
+import { StoreIndex } from "./store-index.js";
+import { StoreManager } from "./store-manager.js";
+import { AppConfigService } from "./app-config.js";
+import { traverseFilterEffect } from "../typeclass/chunk.js";
+import { renderPostMarkdownRow, renderPostsMarkdownHeader } from "../domain/format.js";
+import {
+  FilterCompileError,
+  FilterEvalError,
+  StoreIndexError,
+  StoreIoError
+} from "../domain/errors.js";
+import { defaultStoreConfig } from "../domain/defaults.js";
+import { StorePath, Timestamp } from "../domain/primitives.js";
+import { FilterSettings } from "./filter-settings.js";
+
+export interface MaterializedFilterOutput {
+  readonly name: string;
+  readonly outputPath: string;
+  readonly jsonPath: string | undefined;
+  readonly markdownPath: string | undefined;
+  readonly count: number;
+  readonly updatedAt: Date;
+}
+
+export interface MaterializedStoreOutput {
+  readonly store: string;
+  readonly filters: ReadonlyArray<MaterializedFilterOutput>;
+}
+
+const toStoreIoError = (path: string) => (cause: unknown) =>
+  StoreIoError.make({
+    path: Schema.decodeUnknownSync(StorePath)(path),
+    cause
+  });
+
+const ensureTrailingNewline = (value: string) =>
+  value.endsWith("\n") ? value : `${value}\n`;
+
+const encodePostJson = (post: Post) =>
+  Schema.encodeSync(Schema.parseJson(PostSchema))(post);
+
+const encodeManifestJson = (manifest: unknown) =>
+  Schema.encodeSync(Schema.parseJson(Schema.Unknown))(manifest);
+
+const appendFileString = (
+  fs: FileSystem.FileSystem,
+  targetPath: string,
+  value: string
+) =>
+  fs
+    .writeFileString(targetPath, value, { flag: "a" })
+    .pipe(Effect.mapError(toStoreIoError(targetPath)));
+
+const resolveOutputDir = (
+  path: Path.Path,
+  storeRoot: string,
+  store: StoreRef,
+  outputPath: string
+) => {
+  if (path.isAbsolute(outputPath)) {
+    return outputPath;
+  }
+  return path.join(storeRoot, store.root, outputPath);
+};
+
+const materializeFilter = Effect.fn("OutputManager.materializeFilter")(
+  (store: StoreRef, spec: FilterSpec) =>
+    Effect.gen(function* () {
+      const config = yield* AppConfigService;
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const compiler = yield* FilterCompiler;
+      const runtime = yield* FilterRuntime;
+      const index = yield* StoreIndex;
+      const filterSettings = yield* FilterSettings;
+
+      const outputDir = resolveOutputDir(path, config.storeRoot, store, spec.output.path);
+      yield* fs
+        .makeDirectory(outputDir, { recursive: true })
+        .pipe(Effect.mapError(toStoreIoError(outputDir)));
+
+      const expr = yield* compiler.compile(spec);
+      const stream = index.query(store, StoreQuery.make({ filter: expr }));
+      const predicate = yield* runtime.evaluate(expr);
+      const filtered = stream.pipe(
+        Stream.grouped(50),
+        Stream.mapEffect((batch) =>
+          traverseFilterEffect(batch, predicate, {
+            concurrency: filterSettings.concurrency,
+            batching: true
+          }).pipe(Effect.withRequestBatching(true))
+        ),
+        Stream.mapConcat((chunk) => Chunk.toReadonlyArray(chunk))
+      );
+      const jsonPath = spec.output.json ? path.join(outputDir, "posts.json") : undefined;
+      const markdownPath = spec.output.markdown ? path.join(outputDir, "posts.md") : undefined;
+
+      if (jsonPath) {
+        yield* fs
+          .writeFileString(jsonPath, "[")
+          .pipe(Effect.mapError(toStoreIoError(jsonPath)));
+      }
+      if (markdownPath) {
+        const header = `${renderPostsMarkdownHeader()}\n`;
+        yield* fs
+          .writeFileString(markdownPath, header)
+          .pipe(Effect.mapError(toStoreIoError(markdownPath)));
+      }
+
+      const countRef = yield* Ref.make(0);
+      const jsonFirstRef = jsonPath ? yield* Ref.make(true) : undefined;
+
+      const writeBatch = (batch: Chunk.Chunk<Post>) =>
+        Effect.gen(function* () {
+          const posts = Chunk.toReadonlyArray(batch);
+          if (posts.length === 0) {
+            return;
+          }
+          if (jsonPath && jsonFirstRef) {
+            const isFirst = yield* Ref.get(jsonFirstRef);
+            let first = isFirst;
+            const jsonChunk = posts
+              .map((post) => {
+                const prefix = first ? "" : ",";
+                first = false;
+                return `${prefix}${encodePostJson(post)}`;
+              })
+              .join("");
+            yield* appendFileString(fs, jsonPath, jsonChunk);
+            if (isFirst) {
+              yield* Ref.set(jsonFirstRef, false);
+            }
+          }
+          if (markdownPath) {
+            const markdownChunk = posts
+              .map((post) => `${renderPostMarkdownRow(post)}\n`)
+              .join("");
+            yield* appendFileString(fs, markdownPath, markdownChunk);
+          }
+          yield* Ref.update(countRef, (count) => count + posts.length);
+        });
+
+      yield* filtered.pipe(Stream.grouped(50), Stream.runForEach(writeBatch));
+
+      if (jsonPath && jsonFirstRef) {
+        yield* appendFileString(fs, jsonPath, "]\n");
+      }
+
+      const postsCount = yield* Ref.get(countRef);
+
+      const updatedAt = yield* Clock.currentTimeMillis.pipe(
+        Effect.flatMap((now) =>
+          Schema.decodeUnknown(Timestamp)(new Date(now).toISOString())
+        ),
+        Effect.orDie
+      );
+
+      const manifest = {
+        name: spec.name,
+        count: postsCount,
+        updatedAt: updatedAt.toISOString(),
+        output: {
+          json: jsonPath ? path.basename(jsonPath) : undefined,
+          markdown: markdownPath ? path.basename(markdownPath) : undefined
+        }
+      };
+      const manifestPath = path.join(outputDir, "manifest.json");
+      yield* fs
+        .writeFileString(
+          manifestPath,
+          ensureTrailingNewline(encodeManifestJson(manifest))
+        )
+        .pipe(Effect.mapError(toStoreIoError(manifestPath)));
+
+      return {
+        name: spec.name,
+        outputPath: outputDir,
+        jsonPath,
+        markdownPath,
+        count: postsCount,
+        updatedAt
+      } satisfies MaterializedFilterOutput;
+    })
+);
+
+const materializeFilters = (store: StoreRef, filters: ReadonlyArray<FilterSpec>) =>
+  Effect.forEach(filters, (spec) => materializeFilter(store, spec), {
+    discard: false
+  });
+
+export class OutputManager extends Context.Tag("@skygent/OutputManager")<
+  OutputManager,
+  {
+    readonly materializeStore: (
+      store: StoreRef
+    ) => Effect.Effect<
+      MaterializedStoreOutput,
+      FilterCompileError | FilterEvalError | StoreIndexError | StoreIoError
+    >;
+    readonly materializeFilters: (
+      store: StoreRef,
+      filters: ReadonlyArray<FilterSpec>
+    ) => Effect.Effect<
+      ReadonlyArray<MaterializedFilterOutput>,
+      FilterCompileError | FilterEvalError | StoreIndexError | StoreIoError
+    >;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    OutputManager,
+    Effect.gen(function* () {
+      const appConfig = yield* AppConfigService;
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const compiler = yield* FilterCompiler;
+      const runtime = yield* FilterRuntime;
+      const index = yield* StoreIndex;
+      const manager = yield* StoreManager;
+      const filterSettings = yield* FilterSettings;
+
+      type OutputManagerDeps =
+        | AppConfigService
+        | FileSystem.FileSystem
+        | Path.Path
+        | FilterCompiler
+        | FilterRuntime
+        | StoreIndex
+        | FilterSettings;
+
+      const provideDeps = <A, E>(effect: Effect.Effect<A, E, OutputManagerDeps>) =>
+        effect.pipe(
+          Effect.provideService(AppConfigService, appConfig),
+          Effect.provideService(FileSystem.FileSystem, fs),
+          Effect.provideService(Path.Path, path),
+          Effect.provideService(FilterCompiler, compiler),
+          Effect.provideService(FilterRuntime, runtime),
+          Effect.provideService(StoreIndex, index),
+          Effect.provideService(FilterSettings, filterSettings)
+        );
+
+      const materializeStore = Effect.fn("OutputManager.materializeStore")(
+        (store: StoreRef) =>
+          Effect.gen(function* () {
+            const configOption = yield* manager.getConfig(store.name);
+            const config = Option.getOrElse(configOption, () => defaultStoreConfig);
+
+            const results = yield* provideDeps(materializeFilters(store, config.filters));
+            return {
+              store: store.name,
+              filters: results
+            } satisfies MaterializedStoreOutput;
+          })
+      );
+
+      const materializeFiltersFn = Effect.fn("OutputManager.materializeFilters")(
+        (store: StoreRef, filters: ReadonlyArray<FilterSpec>) =>
+          provideDeps(materializeFilters(store, filters))
+      );
+
+      return OutputManager.of({
+        materializeStore,
+        materializeFilters: materializeFiltersFn
+      });
+    })
+  );
+}

package/src/services/post-parser.ts

@@ -0,0 +1,62 @@
+/**
+ * Post Parser Service Module
+ *
+ * This module provides the PostParser service, which is responsible for parsing
+ * raw post data into normalized Post domain objects. It uses Effect's Schema
+ * validation to ensure type safety and data integrity during the parsing process.
+ *
+ * Key responsibilities:
+ * - Decode unknown/raw data into typed Post objects
+ * - Schema validation and error handling via ParseResult
+ * - Centralized parsing logic for post data ingestion
+ */
+
+import { Context, Effect, Layer, ParseResult, Schema } from "effect";
+import { Post } from "../domain/post.js";
+import { PostFromRaw } from "../domain/raw.js";
+
+/**
+ * Service for parsing raw post data into normalized Post domain objects.
+ *
+ * The PostParser provides a single `parsePost` method that uses Effect's Schema
+ * decoding to transform unknown/raw data into properly typed Post objects.
+ * This ensures type safety and validation during data ingestion.
+ *
+ * @example
+ * ```ts
+ * const program = Effect.gen(function* () {
+ *   const parser = yield* PostParser;
+ *   const rawData = { uri: "at://...", cid: "...", text: "Hello" };
+ *
+ *   const post = yield* parser.parsePost(rawData);
+ *   return post;
+ * });
+ * ```
+ */
+export class PostParser extends Context.Tag("@skygent/PostParser")<
+  PostParser,
+  {
+    /**
+     * Parse raw post data into a normalized Post object.
+     *
+     * This method uses Schema.decodeUnknown with the PostFromRaw schema to:
+     * 1. Validate the input data structure
+     * 2. Transform raw fields into the normalized Post type
+     * 3. Return a typed Post object or a ParseError on failure
+     *
+     * @param raw - The raw, unknown data to parse (typically from an external source)
+     * @returns Effect that resolves to a validated Post object, or fails with ParseError
+     * @throws ParseResult.ParseError if the input data doesn't match the expected schema
+     */
+    readonly parsePost: (raw: unknown) => Effect.Effect<Post, ParseResult.ParseError>;
+  }
+>() {
+  static readonly layer = Layer.succeed(
+    PostParser,
+    PostParser.of({
+      parsePost: Effect.fn("PostParser.parsePost")((raw: unknown) =>
+        Schema.decodeUnknown(PostFromRaw)(raw)
+      )
+    })
+  );
+}