@mepuka/skygent 0.2.0

package/src/services/store-writer.ts

@@ -0,0 +1,200 @@
+import { Clock, Context, Effect, Layer, Option, Random, Schema, SynchronizedRef } from "effect";
+import type * as SqlClient from "@effect/sql/SqlClient";
+import * as SqlSchema from "@effect/sql/SqlSchema";
+import { StoreIoError } from "../domain/errors.js";
+import { type EventLogEntry, PostEvent, PostEventRecord } from "../domain/events.js";
+import { EventId, EventSeq, PostUri } from "../domain/primitives.js";
+import type { StorePath } from "../domain/primitives.js";
+import type { StoreRef } from "../domain/store.js";
+import { StoreDb } from "./store-db.js";
+
+const ULID_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+const MAX_ULID_TIME = 0xffff_ffff_ffff;
+
+class UlidTimeError extends Schema.TaggedError<UlidTimeError>()("UlidTimeError", {
+  time: Schema.String,
+  message: Schema.String
+}) {}
+
+const encodeTime = (time: number) => {
+  if (!Number.isFinite(time) || time < 0 || time > MAX_ULID_TIME) {
+    return Effect.fail(
+      UlidTimeError.make({
+        time: String(time),
+        message: `ULID time out of range: ${time}`
+      })
+    );
+  }
+  return Effect.sync(() => {
+    let value = BigInt(Math.trunc(time));
+    let output = "";
+    for (let i = 0; i < 10; i += 1) {
+      const mod = Number(value % 32n);
+      output = `${ULID_ALPHABET[mod]}${output}`;
+      value = value / 32n;
+    }
+    return output;
+  });
+};
+
+const encodeRandomDigits = (digits: ReadonlyArray<number>) =>
+  digits.map((digit) => ULID_ALPHABET[digit]).join("");
+
+const incrementRandomDigits = (digits: ReadonlyArray<number>) => {
+  const next = digits.slice();
+  for (let i = next.length - 1; i >= 0; i -= 1) {
+    const value = next[i];
+    if (typeof value !== "number") {
+      continue;
+    }
+    if (value === 31) {
+      next[i] = 0;
+      if (i === 0) {
+        return { digits: next, overflow: true } as const;
+      }
+    } else {
+      next[i] = value + 1;
+      return { digits: next, overflow: false } as const;
+    }
+  }
+  return { digits: next, overflow: true } as const;
+};
+
+const eventLogInsertRow = Schema.Struct({
+  event_id: EventId,
+  event_type: Schema.String,
+  post_uri: PostUri,
+  payload_json: Schema.String,
+  created_at: Schema.String,
+  source: Schema.String
+});
+
+const eventLogSeqRow = Schema.Struct({
+  event_seq: EventSeq
+});
+
+const toStoreIoError = (path: StorePath) => (cause: unknown) =>
+  StoreIoError.make({ path, cause });
+
+export class StoreWriter extends Context.Tag("@skygent/StoreWriter")<
+  StoreWriter,
+  {
+    readonly append: (
+      store: StoreRef,
+      event: PostEvent
+    ) => Effect.Effect<EventLogEntry, StoreIoError>;
+    readonly appendWithClient: (
+      client: SqlClient.SqlClient,
+      event: PostEvent
+    ) => Effect.Effect<EventLogEntry, unknown>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    StoreWriter,
+    Effect.gen(function* () {
+      const storeDb = yield* StoreDb;
+      const idState = yield* SynchronizedRef.make({
+        lastTime: 0,
+        lastRandom: [] as ReadonlyArray<number>
+      });
+
+      const nextRandomDigits = () =>
+        Effect.forEach(
+          Array.from({ length: 16 }),
+          () => Random.nextIntBetween(0, 32)
+        );
+
+      const generateEventId = Effect.fn("StoreWriter.generateEventId")(() =>
+        SynchronizedRef.modifyEffect(idState, (state) =>
+          Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis;
+            let time = state.lastTime;
+            let digits = state.lastRandom;
+
+            if (digits.length === 0 || now > state.lastTime) {
+              time = now;
+              digits = yield* nextRandomDigits();
+            } else {
+              const incremented = incrementRandomDigits(digits);
+              if (incremented.overflow) {
+                time = state.lastTime + 1;
+                digits = yield* nextRandomDigits();
+              } else {
+                time = state.lastTime;
+                digits = incremented.digits;
+              }
+            }
+
+            const timeEncoded = yield* encodeTime(time);
+            const id = `${timeEncoded}${encodeRandomDigits(digits)}`;
+            const decoded = yield* Schema.decodeUnknown(EventId)(id);
+            return [
+              decoded,
+              { lastTime: time, lastRandom: digits }
+            ] as const;
+          })
+        )
+      );
+
+      const appendWithClient = Effect.fn("StoreWriter.appendWithClient")(
+        (client: SqlClient.SqlClient, event: PostEvent) =>
+          Effect.gen(function* () {
+            const id = yield* generateEventId();
+            const record = PostEventRecord.make({
+              id,
+              version: 1,
+              event
+            });
+            const payloadJson = yield* Schema.encode(
+              Schema.parseJson(PostEventRecord)
+            )(record);
+            const postUri =
+              record.event._tag === "PostUpsert"
+                ? record.event.post.uri
+                : record.event.uri;
+            const createdAt =
+              record.event.meta.createdAt instanceof Date
+                ? record.event.meta.createdAt.toISOString()
+                : new Date(record.event.meta.createdAt).toISOString();
+
+            const insertEvent = SqlSchema.findOne({
+              Request: eventLogInsertRow,
+              Result: eventLogSeqRow,
+              execute: (row) =>
+                client`INSERT INTO event_log ${client.insert(row)}
+                  RETURNING event_seq`
+            });
+
+            const inserted = yield* insertEvent({
+              event_id: record.id,
+              event_type: record.event._tag,
+              post_uri: postUri,
+              payload_json: payloadJson,
+              created_at: createdAt,
+              source: record.event.meta.source
+            }).pipe(
+              Effect.flatMap(
+                Option.match({
+                  onNone: () =>
+                    Effect.dieMessage("Expected event_seq from event_log insert."),
+                  onSome: (row) => Effect.succeed(row)
+                })
+              )
+            );
+            return { seq: inserted.event_seq, record } satisfies EventLogEntry;
+          })
+      );
+
+      const append = Effect.fn("StoreWriter.append")(
+        (store: StoreRef, event: PostEvent) =>
+          storeDb
+            .withClient(store, (client) =>
+              appendWithClient(client, event).pipe(client.withTransaction)
+            )
+            .pipe(Effect.mapError(toStoreIoError(store.root)))
+      );
+
+      return StoreWriter.of({ append, appendWithClient });
+    })
+  );
+}

package/src/services/sync-checkpoint-store.ts

@@ -0,0 +1,169 @@
+/**
+ * Sync Checkpoint Store Service
+ *
+ * Manages checkpoints for sync operations to enable resumable data synchronization.
+ * Stores the last sync position (cursor/timestamp) for each data source, allowing
+ * sync operations to resume from where they left off rather than starting over.
+ *
+ * Checkpoints are stored per-store and per-data-source combination, using a
+ * prefixed key-value store structure for isolation.
+ *
+ * @module services/sync-checkpoint-store
+ */
+
+import { Context, Effect, Layer, Option, Schema } from "effect";
+import { StoreIoError } from "../domain/errors.js";
+import { DataSourceSchema, SyncCheckpoint, type DataSource, dataSourceKey } from "../domain/sync.js";
+import { EventSeq, Timestamp, type StorePath } from "../domain/primitives.js";
+import type { StoreRef } from "../domain/store.js";
+import { StoreDb } from "./store-db.js";
+
+/**
+ * Converts an error to a StoreIoError with the given path.
+ * @param path - The store path for error context
+ * @returns A function that creates StoreIoError from any cause
+ */
+const toStoreIoError = (path: StorePath) => (cause: unknown) =>
+  StoreIoError.make({ path, cause });
+
+const checkpointRow = Schema.Struct({
+  source_key: Schema.String,
+  source_json: Schema.String,
+  cursor: Schema.NullOr(Schema.String),
+  last_event_seq: Schema.NullOr(EventSeq),
+  filter_hash: Schema.NullOr(Schema.String),
+  updated_at: Schema.String
+});
+
+const decodeSource = (raw: string) =>
+  Schema.decodeUnknown(Schema.parseJson(DataSourceSchema))(raw);
+
+const encodeSource = (source: DataSource) =>
+  Schema.encode(Schema.parseJson(DataSourceSchema))(source);
+
+/**
+ * Service for managing sync operation checkpoints.
+ *
+ * This service provides methods to load and save sync checkpoints, which track
+ * the last successfully processed position for each data source. This enables
+ * resumable sync operations that can pick up where they left off.
+ */
+export class SyncCheckpointStore extends Context.Tag("@skygent/SyncCheckpointStore")<
+  SyncCheckpointStore,
+  {
+    /**
+     * Loads the sync checkpoint for a given store and data source.
+     *
+     * @param store - The store reference to load from
+     * @param source - The data source to get checkpoint for
+     * @returns Effect resolving to Option of SyncCheckpoint, or StoreIoError on failure
+     */
+    readonly load: (
+      store: StoreRef,
+      source: DataSource
+    ) => Effect.Effect<Option.Option<SyncCheckpoint>, StoreIoError>;
+
+    /**
+     * Saves a sync checkpoint for resumable operations.
+     *
+     * @param store - The store reference to save to
+     * @param checkpoint - The checkpoint data to persist
+     * @returns Effect resolving to void, or StoreIoError on failure
+     */
+    readonly save: (
+      store: StoreRef,
+      checkpoint: SyncCheckpoint
+    ) => Effect.Effect<void, StoreIoError>;
+  }
+>() {
+  static readonly layer = Layer.effect(
+    SyncCheckpointStore,
+    Effect.gen(function* () {
+      const storeDb = yield* StoreDb;
+
+      const load = Effect.fn("SyncCheckpointStore.load")(
+        (store: StoreRef, source: DataSource) => {
+          const key = dataSourceKey(source);
+          return storeDb
+            .withClient(store, (client) =>
+              Effect.gen(function* () {
+                const rows = yield* client`SELECT
+                    source_key,
+                    source_json,
+                    cursor,
+                    last_event_seq,
+                    filter_hash,
+                    updated_at
+                  FROM sync_checkpoints
+                  WHERE source_key = ${key}`;
+                if (rows.length === 0) {
+                  return Option.none<SyncCheckpoint>();
+                }
+                const decodedRows = yield* Schema.decodeUnknown(
+                  Schema.Array(checkpointRow)
+                )(rows);
+                const row = decodedRows[0]!;
+                const decodedSource = yield* decodeSource(row.source_json);
+                const updatedAt = yield* Schema.decodeUnknown(Timestamp)(row.updated_at);
+                const checkpoint = SyncCheckpoint.make({
+                  source: decodedSource,
+                  cursor: row.cursor ?? undefined,
+                  lastEventSeq: row.last_event_seq ?? undefined,
+                  filterHash: row.filter_hash ?? undefined,
+                  updatedAt
+                });
+                return Option.some(checkpoint);
+              })
+            )
+            .pipe(Effect.mapError(toStoreIoError(store.root)));
+        }
+      );
+
+      const save = Effect.fn("SyncCheckpointStore.save")(
+        (store: StoreRef, checkpoint: SyncCheckpoint) => {
+          const key = dataSourceKey(checkpoint.source);
+          return storeDb
+            .withClient(store, (client) =>
+              Effect.gen(function* () {
+                const sourceJson = yield* encodeSource(checkpoint.source);
+                const updatedAt = checkpoint.updatedAt.toISOString();
+                const lastSeq = checkpoint.lastEventSeq ?? null;
+                const cursor = checkpoint.cursor ?? null;
+                const filterHash = checkpoint.filterHash ?? null;
+                yield* client`INSERT INTO sync_checkpoints (
+                    source_key,
+                    source_json,
+                    cursor,
+                    last_event_seq,
+                    filter_hash,
+                    updated_at
+                  )
+                  VALUES (
+                    ${key},
+                    ${sourceJson},
+                    ${cursor},
+                    ${lastSeq},
+                    ${filterHash},
+                    ${updatedAt}
+                  )
+                  ON CONFLICT(source_key) DO UPDATE SET
+                    source_json = excluded.source_json,
+                    cursor = excluded.cursor,
+                    filter_hash = excluded.filter_hash,
+                    updated_at = excluded.updated_at,
+                    last_event_seq = CASE
+                      WHEN excluded.last_event_seq IS NULL THEN sync_checkpoints.last_event_seq
+                      WHEN sync_checkpoints.last_event_seq IS NULL THEN excluded.last_event_seq
+                      WHEN excluded.last_event_seq >= sync_checkpoints.last_event_seq THEN excluded.last_event_seq
+                      ELSE sync_checkpoints.last_event_seq
+                    END`;
+              })
+            )
+            .pipe(Effect.mapError(toStoreIoError(store.root)));
+        }
+      );
+
+      return SyncCheckpointStore.of({ load, save });
+    })
+  );
+}