@mepuka/skygent 0.2.0

package/src/services/store-db.ts

@@ -0,0 +1,248 @@
+import { FileSystem, Path } from "@effect/platform";
+import { Context, Effect, Exit, Layer, Ref, Scope } from "effect";
+import * as Reactivity from "@effect/experimental/Reactivity";
+import * as Migrator from "@effect/sql/Migrator";
+import * as MigratorFileSystem from "@effect/sql/Migrator/FileSystem";
+import * as SqlClient from "@effect/sql/SqlClient";
+import { SqliteClient } from "@effect/sql-sqlite-bun";
+import { StoreIoError } from "../domain/errors.js";
+import type { StoreRef } from "../domain/store.js";
+import type { StorePath } from "../domain/primitives.js";
+import { AppConfigService } from "./app-config.js";
+
+/**
+ * Store Database Connection Management Service
+ *
+ * This module provides a centralized SQLite database connection manager for store indexes.
+ * It implements a connection caching/pooling pattern to efficiently manage database connections
+ * across multiple stores while ensuring proper resource cleanup.
+ *
+ * Key features:
+ * - Connection caching: Reuses existing connections to the same store
+ * - Thread-safe client access using semaphores for concurrent operations
+ * - Automatic migration execution on first connection to each store
+ * - Optimized SQLite pragmas for performance (WAL mode, memory-mapped I/O, cache sizing)
+ * - Graceful cleanup on service shutdown via finalizers
+ *
+ * @example
+ * ```typescript
+ * import { Effect } from "effect";
+ * import { StoreDb } from "./services/store-db.js";
+ * import type { StoreRef } from "./domain/store.js";
+ *
+ * const program = Effect.gen(function* () {
+ *   const storeDb = yield* StoreDb;
+ *   const store: StoreRef = { name: "myStore", root: "stores/myStore" };
+ *
+ *   // Execute queries with automatic connection management
+ *   const result = yield* storeDb.withClient(store, (client) =>
+ *     client`SELECT * FROM posts WHERE id = ${postId}`
+ *   );
+ *
+ *   // Remove a store's connection when done
+ *   yield* storeDb.removeClient(store.name);
+ * });
+ *
+ * const runnable = program.pipe(Effect.provide(StoreDb.layer));
+ * ```
+ *
+ * @module services/store-db
+ */
+
+const migrationsDir = decodeURIComponent(
+  new URL("../db/migrations/store-index", import.meta.url).pathname
+);
+
+const toStoreIoError = (path: StorePath) => (cause: unknown) =>
+  StoreIoError.make({ path, cause });
+
+/**
+ * Service for managing SQLite database connections to store indexes.
+ *
+ * Provides connection pooling/caching, automatic migrations, and optimized
+ * SQLite configuration for each store. Connections are lazily created and
+ * cached for reuse, with proper cleanup on service termination.
+ *
+ * @example
+ * ```typescript
+ * // Basic usage - execute a query
+ * const result = yield* storeDb.withClient(store, (client) =>
+ *   client`SELECT COUNT(*) as count FROM posts`
+ * );
+ *
+ * // Batch operations with the same connection
+ * const results = yield* storeDb.withClient(store, (client) =>
+ *   Effect.gen(function* () {
+ *     yield* client`INSERT INTO posts (id, content) VALUES (${id1}, ${content1})`;
+ *     yield* client`INSERT INTO posts (id, content) VALUES (${id2}, ${content2})`;
+ *     return yield* client`SELECT * FROM posts`;
+ *   })
+ * );
+ * ```
+ */
+export class StoreDb extends Context.Tag("@skygent/StoreDb")<
+  StoreDb,
+  {
+    /**
+     * Execute a database operation with a cached client for the specified store.
+     *
+     * This method automatically retrieves an existing connection or creates a new one,
+     * runs the provided operation, and keeps the connection cached for future use.
+     * The connection remains open until explicitly removed via `removeClient` or
+     * service shutdown.
+     *
+     * @param store - Store reference containing name and root path
+     * @param run - Effect function that receives the SQL client and returns a result
+     * @returns Effect containing the operation result, potentially failing with StoreIoError
+     * @example
+     * ```typescript
+     * const posts = yield* storeDb.withClient(store, (client) =>
+     *   client`SELECT * FROM posts LIMIT 10`
+     * );
+     * ```
+     */
+    readonly withClient: <A, E>(
+      store: StoreRef,
+      run: (client: SqlClient.SqlClient) => Effect.Effect<A, E>
+    ) => Effect.Effect<A, StoreIoError | E>;
+
+    /**
+     * Remove a cached database connection for a store.
+     *
+     * Closes the connection gracefully (running PRAGMA optimize first) and
+     * removes it from the cache. Subsequent calls to `withClient` for this
+     * store will create a new connection.
+     *
+     * @param storeName - The name of the store whose connection should be removed
+     * @returns Effect that completes when the connection is closed
+     * @example
+     * ```typescript
+     * // Clean up a store's connection when no longer needed
+     * yield* storeDb.removeClient("myStore");
+     * ```
+     */
+    readonly removeClient: (storeName: string) => Effect.Effect<void>;
+  }
+>() {
+  static readonly layer = Layer.scoped(
+    StoreDb,
+    Effect.gen(function* () {
+      const config = yield* AppConfigService;
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const reactivity = yield* Reactivity.Reactivity;
+
+      type CachedClient = {
+        readonly client: SqlClient.SqlClient;
+        readonly scope: Scope.CloseableScope;
+      };
+      const clients = yield* Ref.make(new Map<string, CachedClient>());
+      const clientLock = yield* Effect.makeSemaphore(1);
+
+      const migrate = Migrator.make({})({
+        loader: MigratorFileSystem.fromFileSystem(migrationsDir)
+      });
+
+      const optimizeClient = (client: SqlClient.SqlClient) =>
+        client`PRAGMA optimize`.pipe(
+          Effect.catchAll((error) =>
+            Effect.logWarning("PRAGMA optimize failed", {
+              error: error instanceof Error ? error.message : String(error)
+            })
+          )
+        );
+
+      const closeCachedClient = ({ client, scope }: CachedClient) =>
+        optimizeClient(client).pipe(Effect.zipRight(Scope.close(scope, Exit.void)));
+
+      const closeAllClients = () =>
+        Ref.get(clients).pipe(
+          Effect.flatMap((current) =>
+            Effect.forEach(current.values(), closeCachedClient, { discard: true })
+          )
+        );
+
+      yield* Effect.addFinalizer(() => closeAllClients());
+
+      const openClient = (store: StoreRef) =>
+        Effect.gen(function* () {
+          const dbPath = path.join(config.storeRoot, store.root, "index.sqlite");
+          const dbDir = path.dirname(dbPath);
+          yield* fs.makeDirectory(dbDir, { recursive: true });
+
+          const clientScope = yield* Scope.make();
+          const client = yield* SqliteClient.make({ filename: dbPath }).pipe(
+            Effect.provideService(Scope.Scope, clientScope),
+            Effect.provideService(Reactivity.Reactivity, reactivity)
+          );
+
+          // Configure SQLite for optimal performance
+          yield* client`PRAGMA busy_timeout = 5000`;
+          yield* client`PRAGMA journal_mode = WAL`;
+          yield* client`PRAGMA synchronous = NORMAL`;
+          yield* client`PRAGMA temp_store = MEMORY`;
+          yield* client`PRAGMA cache_size = -64000`;
+          yield* client`PRAGMA mmap_size = 30000000000`;
+          yield* client`PRAGMA optimize=0x10002`;
+          yield* client`PRAGMA foreign_keys = ON`;
+          yield* migrate.pipe(
+            Effect.provideService(SqlClient.SqlClient, client),
+            Effect.provideService(FileSystem.FileSystem, fs)
+          );
+
+          return { client, scope: clientScope };
+        });
+
+      const getClient = (store: StoreRef) =>
+        Effect.gen(function* () {
+          const cached = (yield* Ref.get(clients)).get(store.name);
+          if (cached) {
+            return cached.client;
+          }
+
+          return yield* clientLock.withPermits(1)(
+            Effect.gen(function* () {
+              const current = yield* Ref.get(clients);
+              const existing = current.get(store.name);
+              if (existing) {
+                return existing.client;
+              }
+
+              const created = yield* openClient(store);
+
+              const next = new Map(current);
+              next.set(store.name, created);
+              yield* Ref.set(clients, next);
+
+              return created.client;
+            })
+          );
+        });
+
+      const withClient = <A, E>(
+        store: StoreRef,
+        run: (client: SqlClient.SqlClient) => Effect.Effect<A, E>
+      ) =>
+        getClient(store).pipe(
+          Effect.mapError(toStoreIoError(store.root)),
+          Effect.flatMap(run)
+        );
+
+      const removeClient = (storeName: string) =>
+        Ref.modify(clients, (current) => {
+          const next = new Map(current);
+          const existing = next.get(storeName);
+          if (existing) {
+            next.delete(storeName);
+          }
+          return [existing, next] as const;
+        }).pipe(
+          Effect.flatMap((existing) =>
+            existing ? closeCachedClient(existing) : Effect.void
+          )
+        );
+
+      return StoreDb.of({ withClient, removeClient });
+    })
+  ).pipe(Layer.provide(Reactivity.layer));
+}

package/src/services/store-event-log.ts

@@ -0,0 +1,285 @@
+import { Context, Effect, Layer, Option, ParseResult, Schema, Stream } from "effect";
+import * as SqlSchema from "@effect/sql/SqlSchema";
+import { StoreIoError } from "../domain/errors.js";
+import { type EventLogEntry, PostEventRecord } from "../domain/events.js";
+import { EventSeq } from "../domain/primitives.js";
+import type { StorePath } from "../domain/primitives.js";
+import type { StoreRef } from "../domain/store.js";
+import { StoreDb } from "./store-db.js";
+
+/**
+ * Store Event Log Service
+ *
+ * This module provides event sourcing capabilities for stores by streaming events
+ * from the SQLite `event_log` table. It implements an append-only event log pattern
+ * for tracking all changes to a store's data, enabling event-driven architectures,
+ * audit trails, and data synchronization.
+ *
+ * Key features:
+ * - Paginated event streaming with configurable batch size (500 events per page)
+ * - Automatic event sequence tracking for resumable streams
+ * - JSON event payload serialization/deserialization
+ * - Event log clearing for data reset scenarios
+ * - Efficient last event sequence retrieval from the event log table
+ *
+ * The service supports the event sourcing pattern where all state changes are stored
+ * as immutable events, allowing for event replay, projections, and temporal queries.
+ *
+ * @example
+ * ```typescript
+ * import { Effect, Stream } from "effect";
+ * import { StoreEventLog } from "./services/store-event-log.js";
+ * import type { StoreRef } from "./domain/store.js";
+ *
+ * const program = Effect.gen(function* () {
+ *   const eventLog = yield* StoreEventLog;
+ *   const store: StoreRef = { name: "myStore", root: "stores/myStore" };
+ *
+ *   // Stream all events from the store
+ *   const events = yield* eventLog.stream(store).pipe(
+ *     Stream.runCollect
+ *   );
+ *
+ *   // Get the last event sequence for resuming streams
+ *   const lastSeq = yield* eventLog.getLastEventSeq(store);
+ *
+ *   // Clear the event log (use with caution)
+ *   yield* eventLog.clear(store);
+ * });
+ *
+ * const runnable = program.pipe(Effect.provide(StoreEventLog.layer));
+ * ```
+ *
+ * @module services/store-event-log
+ */
+
+const pageSize = 500;
+
+const eventLogRow = Schema.Struct({
+  event_seq: EventSeq,
+  payload_json: Schema.String
+});
+
+const lastEventSeqRow = Schema.Struct({
+  value: EventSeq
+});
+
+const decodeEventJson = (raw: string) =>
+  Schema.decodeUnknown(Schema.parseJson(PostEventRecord))(raw);
+
+const toStoreIoError = (path: StorePath) => (cause: unknown) =>
+  StoreIoError.make({ path, cause });
+
+/**
+ * Service for managing and streaming store event logs.
+ *
+ * Provides event sourcing capabilities with paginated streaming, event
+ * tracking, and log management. Events are stored as JSON payloads in an
+ * SQLite table with sequential event numbers for ordering.
+ *
+ * @example
+ * ```typescript
+ * // Stream all events from a store
+ * const events = yield* eventLog.stream(store).pipe(
+ *   Stream.runCollect
+ * );
+ *
+ * // Process events incrementally
+ * yield* eventLog.stream(store).pipe(
+ *   Stream.tap((event) => Effect.sync(() => console.log(event))),
+ *   Stream.runDrain
+ * );
+ * ```
+ */
+export class StoreEventLog extends Context.Tag("@skygent/StoreEventLog")<
+  StoreEventLog,
+  {
+    /**
+     * Stream all events from a store's event log.
+     *
+     * Returns a Stream that emits all `PostEventRecord` events from the store's
+     * event_log table in ascending order by event_seq. Events are fetched in
+     * paginated batches of 500 for memory efficiency. The stream automatically
+     * handles pagination and continues until all events are emitted.
+     *
+     * @param store - Store reference to stream events from
+     * @returns Stream of PostEventRecord events, failing with StoreIoError on database errors
+     * @example
+     * ```typescript
+     * // Collect all events into an array
+     * const allEvents = yield* eventLog.stream(store).pipe(
+     *   Stream.runCollect
+     * );
+     *
+     * // Process events with backpressure
+     * yield* eventLog.stream(store).pipe(
+     *   Stream.grouped(10),
+     *   Stream.tap((batch) => processBatch(batch)),
+     *   Stream.runDrain
+     * );
+     * ```
+     */
+    readonly stream: (
+      store: StoreRef
+    ) => Stream.Stream<EventLogEntry, StoreIoError>;
+
+    /**
+     * Clear all events from a store's event log.
+     *
+     * Deletes all records from the event_log table.
+     * This operation is irreversible and should be used with caution, typically
+     * for data resets or re-initialization scenarios.
+     *
+     * @param store - Store reference whose event log should be cleared
+     * @returns Effect that completes when the event log is cleared, failing with StoreIoError
+     * @example
+     * ```typescript
+     * // Clear the event log before re-syncing
+     * yield* eventLog.clear(store);
+     * // Now ready to start fresh event sourcing
+     * ```
+     */
+    readonly clear: (store: StoreRef) => Effect.Effect<void, StoreIoError>;
+
+    /**
+     * Retrieve the last event sequence from a store's event log.
+     *
+     * Queries the event_log table directly for the highest event_seq.
+     * Returns None if no events exist.
+     *
+     * This is useful for resuming event streams from a known position.
+     *
+     * @param store - Store reference to get the last event sequence from
+     * @returns Effect containing Option of EventSeq (None if no events), failing with StoreIoError
+     * @example
+     * ```typescript
+     * // Get last event sequence for incremental processing
+     * const lastSeq = yield* eventLog.getLastEventSeq(store);
+     *
+     * // Use with Option.match to handle empty event log
+     * yield* Option.match(lastId, {
+     *   onNone: () => syncAllEvents(),
+     *   onSome: (id) => syncEventsSince(id)
+     * });
+     * ```
+     */
+    readonly getLastEventSeq: (
+      store: StoreRef
+    ) => Effect.Effect<Option.Option<EventSeq>, StoreIoError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    StoreEventLog,
+    Effect.gen(function* () {
+      const storeDb = yield* StoreDb;
+
+      const stream = (store: StoreRef) =>
+        Stream.unwrap(
+          storeDb.withClient(store, (client) =>
+            Effect.gen(function* () {
+              const firstPage = SqlSchema.findAll({
+                Request: Schema.Void,
+                Result: eventLogRow,
+                execute: () =>
+                  client`SELECT event_seq, payload_json
+                    FROM event_log
+                    ORDER BY event_seq ASC
+                    LIMIT ${pageSize}`
+              });
+              const nextPage = SqlSchema.findAll({
+                Request: EventSeq,
+                Result: eventLogRow,
+                execute: (after) =>
+                  client`SELECT event_seq, payload_json
+                    FROM event_log
+                    WHERE event_seq > ${after}
+                    ORDER BY event_seq ASC
+                    LIMIT ${pageSize}`
+              });
+
+              const decodeRows = (rows: ReadonlyArray<typeof eventLogRow.Type>) =>
+                Effect.forEach(
+                  rows,
+                  (row) =>
+                    decodeEventJson(row.payload_json).pipe(
+                      Effect.map((record) => ({
+                        seq: row.event_seq,
+                        record
+                      }) satisfies EventLogEntry)
+                    ),
+                  { discard: false }
+                );
+
+              const toPage = (
+                rows: ReadonlyArray<typeof eventLogRow.Type>
+              ): Effect.Effect<
+                Option.Option<readonly [ReadonlyArray<EventLogEntry>, Option.Option<EventSeq>]>,
+                ParseResult.ParseError
+              > =>
+                rows.length === 0
+                  ? Effect.succeed(Option.none())
+                  : decodeRows(rows).pipe(
+                      Effect.map((records) => {
+                        const lastSeq = rows[rows.length - 1]!.event_seq;
+                        return Option.some([
+                          records,
+                          Option.some(lastSeq)
+                        ] as const);
+                      })
+                    );
+
+              return Stream.unfoldEffect(
+                Option.none<EventSeq>(),
+                (cursor) =>
+                  Option.match(cursor, {
+                    onNone: () => firstPage(undefined).pipe(Effect.flatMap(toPage)),
+                    onSome: (after) => nextPage(after).pipe(Effect.flatMap(toPage))
+                  })
+              ).pipe(
+                Stream.mapConcat((records) => records),
+                Stream.mapError(toStoreIoError(store.root))
+              );
+            })
+          ).pipe(Effect.mapError(toStoreIoError(store.root)))
+        );
+
+      const clear = Effect.fn("StoreEventLog.clear")((store: StoreRef) =>
+        storeDb
+          .withClient(store, (client) =>
+            Effect.gen(function* () {
+              yield* client`DELETE FROM event_log`;
+              yield* client`DELETE FROM event_log_meta`;
+            }).pipe(Effect.asVoid)
+          )
+          .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      const getLastEventSeq = Effect.fn("StoreEventLog.getLastEventSeq")(
+        (store: StoreRef) =>
+          storeDb
+            .withClient(store, (client) => {
+              const findLast = SqlSchema.findAll({
+                Request: Schema.Void,
+                Result: lastEventSeqRow,
+                execute: () =>
+                  client`SELECT event_seq as value
+                    FROM event_log
+                    ORDER BY event_seq DESC
+                    LIMIT 1`
+              });
+
+              return findLast(undefined).pipe(
+                Effect.flatMap((rows) =>
+                  rows.length > 0
+                    ? Effect.succeed(Option.some(rows[0]!.value))
+                    : Effect.succeed(Option.none<EventSeq>())
+                )
+              );
+            })
+            .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      return StoreEventLog.of({ stream, clear, getLastEventSeq });
+    })
+  );
+}