@mepuka/skygent 0.2.0 → 0.3.1

package/src/services/sync-engine.ts

@@ -59,8 +59,9 @@
  * @module services/sync-engine
  */
 
-import { Clock, Context, Duration, Effect, Layer, Option, Ref, Schedule, Schema, Stream } from "effect";
+import { Chunk, Clock, Context, Duration, Effect, Fiber, Layer, Option, Ref, Schedule, Schema, Stream } from "effect";
 import { messageFromCause } from "./shared.js";
+import type { BskyError } from "../domain/errors.js";
 import { FilterRuntime } from "./filter-runtime.js";
 import { PostParser } from "./post-parser.js";
 import { StoreCommitter } from "./store-commit.js";
@@ -68,6 +69,7 @@ import { BskyClient } from "./bsky-client.js";
 import type { FilterExpr } from "../domain/filter.js";
 import { filterExprSignature } from "../domain/filter.js";
 import { EventMeta, PostUpsert } from "../domain/events.js";
+import type { EventLogEntry } from "../domain/events.js";
 import type { Post } from "../domain/post.js";
 import { EventSeq, Timestamp } from "../domain/primitives.js";
 import type { RawPost } from "../domain/raw.js";
@@ -154,7 +156,7 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
       source: DataSource,
       target: StoreRef,
       filter: FilterExpr,
-      options?: { readonly policy?: SyncUpsertPolicy }
+      options?: { readonly policy?: SyncUpsertPolicy; readonly limit?: number }
     ) => Effect.Effect<SyncResult, SyncError>;
     readonly watch: (config: WatchConfig) => Stream.Stream<SyncEvent, SyncError>;
   }
@@ -171,7 +173,12 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
       const settings = yield* SyncSettings;
 
       const sync = Effect.fn("SyncEngine.sync")(
-        (source: DataSource, target: StoreRef, filter: FilterExpr, options?: { readonly policy?: SyncUpsertPolicy }) =>
+        (
+          source: DataSource,
+          target: StoreRef,
+          filter: FilterExpr,
+          options?: { readonly policy?: SyncUpsertPolicy; readonly limit?: number }
+        ) =>
           Effect.gen(function* () {
             const predicate = yield* runtime
               .evaluateWithMetadata(filter)
@@ -200,29 +207,10 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                 )
               );
 
-            const storePost = (post: Post) =>
-              Effect.gen(function* () {
-                const meta = yield* makeMeta();
-                const event = PostUpsert.make({ post, meta });
-                if (policy === "refresh") {
-                  const record = yield* committer
-                    .appendUpsert(target, event)
-                    .pipe(
-                      Effect.mapError(
-                        toSyncError("store", "Failed to append event")
-                      )
-                    );
-                  return Option.some(record.seq);
-                }
-                const stored = yield* committer
-                  .appendUpsertIfMissing(target, event)
-                  .pipe(
-                    Effect.mapError(
-                      toSyncError("store", "Failed to append event")
-                    )
-                  );
-                return Option.map(stored, (entry) => entry.seq);
-              });
+            const buildUpsert = (post: Post) =>
+              makeMeta().pipe(
+                Effect.map((meta) => PostUpsert.make({ post, meta }))
+              );
 
             const prepareRaw = (raw: RawPost): Effect.Effect<PreparedOutcome, SyncError> =>
               parser.parsePost(raw).pipe(
@@ -244,23 +232,53 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                 )
               );
 
-            const applyPrepared = (
-              prepared: PreparedOutcome
-            ): Effect.Effect<SyncOutcome, SyncError> =>
+            const commitStoreEvents = (events: ReadonlyArray<PostUpsert>) => {
+              if (events.length === 0) {
+                return Effect.succeed(
+                  [] as ReadonlyArray<Option.Option<EventLogEntry>>
+                );
+              }
+              const commit = policy === "refresh"
+                ? committer
+                    .appendUpserts(target, events)
+                    .pipe(Effect.map((entries) => entries.map(Option.some)))
+                : committer.appendUpsertsIfMissing(target, events);
+              return commit.pipe(
+                Effect.mapError(
+                  toSyncError("store", "Failed to append events")
+                )
+              );
+            };
+
+            const applyPreparedBatch = (
+              preparedBatch: ReadonlyArray<PreparedOutcome>
+            ): Effect.Effect<ReadonlyArray<SyncOutcome>, SyncError> =>
               Effect.gen(function* () {
-                switch (prepared._tag) {
-                  case "Skip":
-                    return skippedOutcome;
-                  case "Error":
-                    return { _tag: "Error", error: prepared.error } as const;
-                  case "Store": {
-                    const stored = yield* storePost(prepared.post);
-                    return Option.match(stored, {
-                      onNone: () => skippedOutcome,
-                      onSome: (eventSeq) => ({ _tag: "Stored", eventSeq } as const)
-                    });
+                const storeItems = preparedBatch.filter(
+                  (item): item is Extract<PreparedOutcome, { _tag: "Store" }> =>
+                    item._tag === "Store"
+                );
+                const events = yield* Effect.forEach(storeItems, (item) =>
+                  buildUpsert(item.post)
+                );
+                const storedEntries = yield* commitStoreEvents(events);
+                let storeIndex = 0;
+                return preparedBatch.map((prepared) => {
+                  switch (prepared._tag) {
+                    case "Skip":
+                      return skippedOutcome;
+                    case "Error":
+                      return { _tag: "Error", error: prepared.error } as const;
+                    case "Store": {
+                      const entry = storedEntries[storeIndex++] ?? Option.none();
+                      return Option.match(entry, {
+                        onNone: () => skippedOutcome,
+                        onSome: (record) =>
+                          ({ _tag: "Stored", eventSeq: record.seq } as const)
+                      });
+                    }
                   }
-                }
+                });
               });
 
             const initial = SyncResultMonoid.empty;
@@ -273,41 +291,43 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
             const cursorOption = Option.flatMap(activeCheckpoint, (value) =>
               Option.fromNullable(value.cursor)
             );
+            const pageLimit = settings.pageLimit;
 
-            const stream = (() => {
+            let stream = (() => {
               switch (source._tag) {
                 case "Timeline":
                   return client.getTimeline(
                     Option.match(cursorOption, {
-                      onNone: () => undefined,
-                      onSome: (value) => ({ cursor: value })
+                      onNone: () => ({ limit: pageLimit }),
+                      onSome: (value) => ({ cursor: value, limit: pageLimit })
                     })
                   );
                 case "Feed":
                   return client.getFeed(
                     source.uri,
                     Option.match(cursorOption, {
-                      onNone: () => undefined,
-                      onSome: (value) => ({ cursor: value })
+                      onNone: () => ({ limit: pageLimit }),
+                      onSome: (value) => ({ cursor: value, limit: pageLimit })
                     })
                   );
                 case "List":
                   return client.getListFeed(
                     source.uri,
                     Option.match(cursorOption, {
-                      onNone: () => undefined,
-                      onSome: (value) => ({ cursor: value })
+                      onNone: () => ({ limit: pageLimit }),
+                      onSome: (value) => ({ cursor: value, limit: pageLimit })
                     })
                   );
                 case "Notifications":
                   return client.getNotifications(
                     Option.match(cursorOption, {
-                      onNone: () => undefined,
-                      onSome: (value) => ({ cursor: value })
+                      onNone: () => ({ limit: pageLimit }),
+                      onSome: (value) => ({ cursor: value, limit: pageLimit })
                     })
                   );
                 case "Author":
                   const authorOptions = {
+                    limit: pageLimit,
                     ...(source.filter !== undefined
                       ? { filter: source.filter }
                       : {}),
@@ -346,7 +366,10 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                     })
                   );
               }
-            })();
+            })() as Stream.Stream<RawPost, BskyError | SyncError>;
+            if (options?.limit !== undefined) {
+              stream = stream.pipe(Stream.take(options.limit));
+            }
 
             type SyncState = {
               readonly result: SyncResult;
@@ -403,6 +426,7 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
             };
 
             const startTime = yield* Clock.currentTimeMillis;
+            const progressIntervalMs = 5000;
             const initialState: SyncState = {
               result: initial,
               lastEventSeq: Option.none<EventSeq>(),
@@ -415,6 +439,33 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
               lastCheckpointAt: startTime
             };
             const stateRef = yield* Ref.make(initialState);
+            const heartbeat = Effect.gen(function* () {
+              while (true) {
+                yield* Effect.sleep(Duration.millis(progressIntervalMs));
+                const now = yield* Clock.currentTimeMillis;
+                const state = yield* Ref.get(stateRef);
+                if (now - state.lastReportAt < progressIntervalMs) {
+                  continue;
+                }
+                const elapsedMs = now - startTime;
+                const rate = elapsedMs > 0 ? state.processed / (elapsedMs / 1000) : 0;
+                yield* reporter.report(
+                  SyncProgress.make({
+                    processed: state.processed,
+                    stored: state.stored,
+                    skipped: state.skipped,
+                    errors: state.errors,
+                    elapsedMs,
+                    rate
+                  })
+                );
+                yield* Ref.update(stateRef, (current) => ({
+                  ...current,
+                  lastReportAt: now
+                }));
+              }
+            });
+            const heartbeatFiber = yield* Effect.fork(heartbeat);
 
             const state = yield* stream.pipe(
               Stream.mapError(toSyncError("source", "Source stream failed")),
@@ -422,47 +473,48 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                 concurrency: settings.concurrency,
                 unordered: false
               }),
+              Stream.grouped(settings.batchSize),
               Stream.runFoldEffect(
                 initialState,
-                (state, prepared) =>
+                (state, preparedChunk) =>
                   Effect.gen(function* () {
-                    const outcome = yield* applyPrepared(prepared);
-                    const delta = (() => {
+                    const preparedBatch = Chunk.toReadonlyArray(preparedChunk);
+                    const outcomes = yield* applyPreparedBatch(preparedBatch);
+                    let storedDelta = 0;
+                    let skippedDelta = 0;
+                    let errorDelta = 0;
+                    let lastStoredSeq = Option.none<EventSeq>();
+                    const errorList: Array<SyncError> = [];
+                    for (const outcome of outcomes) {
                       switch (outcome._tag) {
                         case "Stored":
-                          return SyncResult.make({
-                            postsAdded: 1,
-                            postsDeleted: 0,
-                            postsSkipped: 0,
-                            errors: []
-                          });
+                          storedDelta += 1;
+                          lastStoredSeq = Option.some(outcome.eventSeq);
+                          break;
                         case "Skipped":
-                          return SyncResult.make({
-                            postsAdded: 0,
-                            postsDeleted: 0,
-                            postsSkipped: 1,
-                            errors: []
-                          });
+                          skippedDelta += 1;
+                          break;
                         case "Error":
-                          return SyncResult.make({
-                            postsAdded: 0,
-                            postsDeleted: 0,
-                            postsSkipped: 1,
-                            errors: [outcome.error]
-                          });
+                          skippedDelta += 1;
+                          errorDelta += 1;
+                          errorList.push(outcome.error);
+                          break;
                       }
-                    })();
-
-                    const processed = state.processed + 1;
-                    const stored =
-                      state.stored + (outcome._tag === "Stored" ? 1 : 0);
-                    const skipped =
-                      state.skipped + (outcome._tag === "Skipped" ? 1 : 0);
-                    const errors =
-                      state.errors + (outcome._tag === "Error" ? 1 : 0);
+                    }
+                    const delta = SyncResult.make({
+                      postsAdded: storedDelta,
+                      postsDeleted: 0,
+                      postsSkipped: skippedDelta,
+                      errors: errorList
+                    });
+
+                    const processed = state.processed + preparedBatch.length;
+                    const stored = state.stored + storedDelta;
+                    const skipped = state.skipped + skippedDelta;
+                    const errors = state.errors + errorDelta;
                     const now = yield* Clock.currentTimeMillis;
                     const shouldReport =
-                      processed % 100 === 0 || now - state.lastReportAt >= 5000;
+                      processed % 100 === 0 || now - state.lastReportAt >= progressIntervalMs;
                     if (shouldReport) {
                       const elapsedMs = now - startTime;
                       const rate =
@@ -479,16 +531,21 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                       );
                     }
 
-                    const nextCursor = prepared.pageCursor
-                      ? Option.some(prepared.pageCursor)
-                      : state.latestCursor;
+                    const nextCursor = preparedBatch.reduce(
+                      (cursor, prepared) =>
+                        prepared.pageCursor
+                          ? Option.some(prepared.pageCursor)
+                          : cursor,
+                      state.latestCursor
+                    );
+                    const nextLastEventSeq =
+                      Option.isSome(lastStoredSeq)
+                        ? lastStoredSeq
+                        : state.lastEventSeq;
 
                     const nextState: SyncState = {
                       result: SyncResultMonoid.combine(state.result, delta),
-                      lastEventSeq:
-                        outcome._tag === "Stored"
-                          ? Option.some(outcome.eventSeq)
-                          : state.lastEventSeq,
+                      lastEventSeq: nextLastEventSeq,
                       latestCursor: nextCursor,
                       processed,
                       stored,
@@ -516,6 +573,9 @@ export class SyncEngine extends Context.Tag("@skygent/SyncEngine")<
                   })
               ),
               Effect.withRequestBatching(true),
+              Effect.ensuring(
+                Fiber.interrupt(heartbeatFiber)
+              ),
               Effect.ensuring(
                 Ref.get(stateRef).pipe(
                   Effect.flatMap((state) =>

package/src/services/sync-reporter.ts

@@ -5,12 +5,14 @@ export class SyncReporter extends Context.Tag("@skygent/SyncReporter")<
   SyncReporter,
   {
     readonly report: (progress: SyncProgress) => Effect.Effect<void>;
+    readonly warn: (message: string, data?: Record<string, unknown>) => Effect.Effect<void>;
   }
 >() {
   static readonly layer = Layer.succeed(
     SyncReporter,
     SyncReporter.of({
-      report: () => Effect.void
+      report: () => Effect.void,
+      warn: () => Effect.void
     })
   );
 }

package/src/services/sync-settings.ts

@@ -5,6 +5,8 @@ export type SyncSettingsValue = {
   readonly checkpointEvery: number;
   readonly checkpointIntervalMs: number;
   readonly concurrency: number;
+  readonly batchSize: number;
+  readonly pageLimit: number;
 };
 
 type SyncSettingsOverridesValue = Partial<SyncSettingsValue>;
@@ -36,11 +38,19 @@ export class SyncSettings extends Context.Tag("@skygent/SyncSettings")<
       const concurrency = yield* Config.integer("SKYGENT_SYNC_CONCURRENCY").pipe(
         Config.withDefault(5)
       );
+      const batchSize = yield* Config.integer("SKYGENT_SYNC_BATCH_SIZE").pipe(
+        Config.withDefault(100)
+      );
+      const pageLimit = yield* Config.integer("SKYGENT_SYNC_PAGE_LIMIT").pipe(
+        Config.withDefault(100)
+      );
 
       const merged = {
         checkpointEvery,
         checkpointIntervalMs,
         concurrency,
+        batchSize,
+        pageLimit,
         ...pickDefined(overrides as Record<string, unknown>)
       } as SyncSettingsValue;
 
@@ -65,6 +75,20 @@ export class SyncSettings extends Context.Tag("@skygent/SyncSettings")<
       if (concurrencyError) {
         return yield* concurrencyError;
       }
+      const batchSizeError = validatePositive(
+        "SKYGENT_SYNC_BATCH_SIZE",
+        merged.batchSize
+      );
+      if (batchSizeError) {
+        return yield* batchSizeError;
+      }
+      const pageLimitError = validatePositive(
+        "SKYGENT_SYNC_PAGE_LIMIT",
+        merged.pageLimit
+      );
+      if (pageLimitError) {
+        return yield* pageLimitError;
+      }
 
       return SyncSettings.of(merged);
     })