@useairfoil/connector-kit 0.6.0 → 0.6.1

package/src/ingestion/engine.ts

@@ -1,8 +1,9 @@
-import { DateTime, Effect, Layer, Queue } from "effect";
-import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/unstable/http";
-import * as Observability from "effect/unstable/observability";
+import { Array as Arr, DateTime, Effect, Layer, Order, Queue } from "effect";
+import { HttpRouter, type HttpServer, HttpServerResponse } from "effect/http";
+import * as Observability from "effect/observability";
 
 import type {
+  ChangesFeed,
   ConnectorDefinition,
   Cursor,
   ResourceDefinition,
@@ -12,11 +13,11 @@ import type {
 import type { StateOperation, StateSource } from "../state-store/schema";
 
 import { ConnectorError } from "../errors";
+import { ingestBatch } from "../ingestor/instrumented";
+import { Ingestor } from "../ingestor/service";
 import * as Metrics from "../metrics";
-import { publishBatch } from "../publisher/instrumented";
-import { Publisher } from "../publisher/service";
 import { StateStore } from "../state-store";
-import { deriveSyncState, normalizeCursor } from "../state-store/state";
+import { deriveSyncState, hasChanges, normalizeCursor } from "../state-store/state";
 import { statusRoute } from "../status";
 import { Attr, SpanName, annotateError } from "../telemetry";
 import { router, WebhookQueue, type QueuedWebhookBatch } from "../webhook/server";
@@ -44,47 +45,70 @@ type RunWebhookOptions = RunOptions & {
 };
 
 /**
- * Runs all connector ingestion loops. Callers provide the Publisher and
+ * Runs all connector ingestion loops. Callers provide the Ingestor and
  * StateStore layers; webhook mode additionally requires an HttpServer layer.
  */
 export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   connector: ConnectorDefinition<Resources>,
   options?: RunNoWebhookOptions,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor>;
 export function run<const Resources extends ReadonlyArray<ResourceDefinition>>(
   connector: ConnectorDefinition<Resources>,
   options: RunWebhookOptions,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
-export function run(connector: ConnectorDefinition, options?: RunOptions) {
-  const queueLayer = makeWebhookQueueLayer(connector.name);
-  const runtimeLayer = options?.webhook
-    ? Layer.mergeAll(
-        queueLayer,
-        makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
-      )
-    : Layer.empty;
-
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor | HttpServer.HttpServer>;
+export function run(definition: ConnectorDefinition, options?: RunOptions) {
   return Effect.gen(function* () {
-    const initialCutoff =
-      options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
-    yield* Effect.logInfo("Connector started").pipe(
-      Effect.annotateLogs({
-        [Attr.connectorName]: connector.name,
-        resources: connector.resources.length,
-      }),
-    );
+    // Check the whole feed, even the resources that don't run.
+    if (definition.changes) yield* validateFeed(definition, definition.changes);
+    const ingestor = yield* Ingestor;
+    const connector = selectResources(definition, ingestor.resourceNames);
+    const queueLayer = makeWebhookQueueLayer(connector.name);
+    const runtimeLayer = options?.webhook
+      ? Layer.mergeAll(
+          queueLayer,
+          makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),
+        )
+      : Layer.empty;
 
-    const sourceRun = runIngestion(connector, initialCutoff);
-    const webhookRun = options?.webhook
-      ? runWebhookRuntime(connector, options.webhook.routes)
-      : Effect.void;
+    return yield* Effect.gen(function* () {
+      const initialCutoff =
+        options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+      yield* Effect.logInfo("Connector started").pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resources: connector.resources.length,
+        }),
+      );
 
-    return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
-      Effect.asVoid,
-    );
-  }).pipe(Effect.provide(runtimeLayer));
+      const sourceRun = runIngestion(connector, initialCutoff);
+      const webhookRun = options?.webhook
+        ? runWebhookRuntime(connector, options.webhook.routes)
+        : Effect.void;
+
+      return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(
+        Effect.asVoid,
+      );
+    }).pipe(Effect.provide(runtimeLayer));
+  });
 }
 
+const selectResources = (
+  connector: ConnectorDefinition,
+  names: ReadonlyArray<string> | undefined,
+): ConnectorDefinition => {
+  if (names === undefined) return connector;
+  const isSelected = (resource: ResourceDefinition) => names.includes(resource.name);
+  const { changes, ...rest } = connector;
+  const feedResources = changes?.resources.filter(isSelected) ?? [];
+  return {
+    ...rest,
+    resources: connector.resources.filter(isSelected),
+    ...(changes && feedResources.length > 0
+      ? { changes: { ...changes, resources: feedResources } }
+      : {}),
+  };
+};
+
 const makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>
   Layer.effect(WebhookQueue)(
     Effect.gen(function* () {
@@ -111,7 +135,7 @@ const runWebhookQueueConsumer = (connector: ConnectorDefinition) =>
       const batch = yield* Queue.take(queue.queue);
       const depth = yield* Queue.size(queue.queue);
       yield* Metrics.setWebhookQueueDepth(connector.name, depth);
-      yield* publishBatch({
+      yield* ingestBatch({
         connector: connector.name,
         resource: batch.resource,
         source: "webhook",
@@ -134,7 +158,10 @@ const makeWebhookServerLayer = (
   const metricsPath = options.metricsPath === undefined ? "/metrics" : options.metricsPath;
   const statusPath = options.statusPath === undefined ? "/status" : options.statusPath;
   const app = Layer.mergeAll(
-    router(options.routes, { connectorName: connector.name }),
+    router(options.routes, {
+      connectorName: connector.name,
+      resourceNames: connector.resources.map((resource) => resource.name),
+    }),
     HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))),
     ...(metricsPath === false
       ? []
@@ -150,14 +177,57 @@ const makeWebhookServerLayer = (
 const runIngestion = (
   connector: ConnectorDefinition,
   initialCutoff: Cursor.Value,
-): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>
-  Effect.forEach(
-    connector.resources,
-    (resource) => runResourceSources(connector, resource, initialCutoff),
-    {
-      concurrency: "unbounded",
-    },
-  ).pipe(Effect.asVoid);
+): Effect.Effect<void, ConnectorError, StateStore | Ingestor> =>
+  Effect.gen(function* () {
+    const feed = connector.changes;
+    yield* Effect.all(
+      [
+        Effect.forEach(
+          connector.resources,
+          (resource) => runResourceSources(connector, resource, initialCutoff),
+          { concurrency: "unbounded" },
+        ),
+        feed ? runChangesFeed(connector, feed, initialCutoff) : Effect.void,
+      ],
+      { concurrency: "unbounded" },
+    );
+  });
+
+const bySavedAt = Order.mapInput(
+  Order.String,
+  (state: NonNullable<ResourceState["changes"]>) => state.lastSuccessAt ?? "",
+);
+
+const validateFeed = (connector: ConnectorDefinition, feed: ChangesFeed) =>
+  Effect.forEach(feed.resources, (member, index) => {
+    if (feed.resources.findIndex((item) => item.name === member.name) !== index) {
+      return Effect.fail(
+        new ConnectorError({ message: `Resource ${member.name} is in the changes feed twice` }),
+      );
+    }
+    const resource = connector.resources.find((item) => item.name === member.name);
+    if (!resource) {
+      return Effect.fail(
+        new ConnectorError({
+          message: `Changes feed resource ${member.name} is not a resource of connector ${connector.name}`,
+        }),
+      );
+    }
+    if (resource.changes) {
+      return Effect.fail(
+        new ConnectorError({
+          message: `Resource ${member.name} has its own changes and is also in the changes feed`,
+        }),
+      );
+    }
+    return Effect.void;
+  }).pipe(
+    Effect.andThen(
+      feed.resources.length === 0
+        ? Effect.fail(new ConnectorError({ message: "The changes feed has no resources" }))
+        : Effect.void,
+    ),
+  );
 
 const initializeResourceState = (
   existing: ResourceState | undefined,
@@ -207,9 +277,9 @@ const runResourceSources = (
   return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(
     Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })),
     Effect.asVoid,
-    Effect.catch(() =>
+    Effect.catch((error) =>
       Metrics.setSyncState({ connector: connector.name, resource: resource.name }, "error").pipe(
-        Effect.andThen(Effect.logError("Connector resource initialization failed")),
+        Effect.andThen(Effect.logError("Connector resource initialization failed", error)),
         Effect.annotateLogs({
           [Attr.connectorName]: connector.name,
           resource: resource.name,
@@ -250,8 +320,8 @@ const isolateSourceFailure = <A, R>(
 ): Effect.Effect<A, never, R> =>
   // Only typed source errors are isolated. Defects and interruption still stop the runtime.
   effect.pipe(
-    Effect.catch(() =>
-      Effect.logError("Connector source parked").pipe(
+    Effect.catch((error) =>
+      Effect.logError("Connector source parked", error).pipe(
         Effect.annotateLogs({
           [Attr.connectorName]: connector.name,
           resource: resource.name,
@@ -270,8 +340,9 @@ const refreshRuntimeStatus = Effect.fnUntraced(function* (
   const state = yield* getInitializedState(resource.name, initialCutoff);
   yield* Metrics.setSyncState(
     { connector: connector.name, resource: resource.name },
-    deriveSyncState(resource, state),
+    deriveSyncState(connector, resource, state),
   );
+
   return state;
 });
 
@@ -283,15 +354,16 @@ const initializeRuntimeStatus = Effect.fnUntraced(function* (
   const state = yield* getInitializedState(resource.name, initialCutoff);
   yield* Metrics.setSyncState(
     { connector: connector.name, resource: resource.name },
-    deriveSyncState(resource, state),
+    deriveSyncState(connector, resource, state),
   );
+
   if (resource.backfill) {
     yield* Metrics.setLastSuccessTimestamp(
       { connector: connector.name, resource: resource.name, source: "backfill" },
       state.backfill?.lastSuccessAt,
     );
   }
-  if (resource.changes) {
+  if (hasChanges(connector, resource)) {
     yield* Metrics.setLastSuccessTimestamp(
       { connector: connector.name, resource: resource.name, source: "changes" },
       state.changes?.lastSuccessAt,
@@ -328,15 +400,15 @@ const runBackfill = Effect.fnUntraced(function* (
       })
       .pipe(recordResourceError(connector, resource, "backfill", "fetch"));
 
-    yield* publishBatch({
+    yield* ingestBatch({
       connector: connector.name,
       resource: resource.name,
       source: "backfill",
       batch: {
         cursor: page.nextPageCursor ?? backfill.cutoff,
-        mutations: page.mutations,
+        rows: page.rows,
       },
-    }).pipe(recordResourceError(connector, resource, "backfill", "publish"));
+    }).pipe(recordResourceError(connector, resource, "backfill", "ingest"));
 
     const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
     state = {
@@ -349,7 +421,7 @@ const runBackfill = Effect.fnUntraced(function* (
       },
     };
 
-    // Checkpoint only after publish succeeds so a crash cannot skip an
+    // Checkpoint only after ingestion succeeds so a crash cannot skip an
     // unacknowledged page; replay therefore remains at-least-once.
     yield* Effect.withSpan(
       store
@@ -397,18 +469,18 @@ const runChanges = Effect.fnUntraced(function* (
       .fetch({ cursor })
       .pipe(recordResourceError(connector, resource, "changes", "fetch"));
 
-    yield* publishBatch({
+    yield* ingestBatch({
       connector: connector.name,
       resource: resource.name,
       source: "changes",
       batch: {
         cursor: page.cursor,
-        mutations: page.mutations,
+        rows: page.rows,
       },
-    }).pipe(recordResourceError(connector, resource, "changes", "publish"));
+    }).pipe(recordResourceError(connector, resource, "changes", "ingest"));
 
     const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
-    // Advance the durable cursor only after Wings accepts the page.
+    // Advance the durable cursor only after Wings acknowledges the page.
     yield* Effect.withSpan(
       store
         .setChangesState(resource.name, {
@@ -433,6 +505,114 @@ const runChanges = Effect.fnUntraced(function* (
       yield* refreshRuntimeStatus(connector, resource, initialCutoff);
     }
 
-    yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+    if (!page.hasMore) {
+      yield* Effect.sleep(resource.changes.interval ?? "1 minute");
+    }
   }
 });
+
+// Every member resource stores the feed cursor. All rows are ingested before any
+// cursor is written. If a crash stops the writes partway, the next run starts
+// from the oldest cursor.
+const runChangesFeed = (
+  connector: ConnectorDefinition,
+  feed: ChangesFeed,
+  initialCutoff: Cursor.Value,
+): Effect.Effect<void, never, StateStore | Ingestor> =>
+  Effect.gen(function* () {
+    const store = yield* StateStore;
+    const members = feed.resources;
+    const selected = new Set(members.map((resource) => resource.name));
+    // Any failure stops the whole feed, so it is an error for every resource in it.
+    const recordFeedError =
+      (operation: StateOperation) =>
+      <A, R>(effect: Effect.Effect<A, ConnectorError, R>) =>
+        effect.pipe(
+          Effect.tapError(() =>
+            Effect.forEach(
+              members,
+              (resource) =>
+                Metrics.setSyncState(
+                  { connector: connector.name, resource: resource.name },
+                  "error",
+                ).pipe(Effect.andThen(store.setResourceError(resource.name, "changes", operation))),
+              { discard: true },
+            ),
+          ),
+        );
+
+    while (true) {
+      const states = yield* Effect.forEach(members, (resource) =>
+        store.getResourceState(resource.name),
+      );
+      // A resource that was off has an older cursor. Resume from the oldest, so it
+      // gets the changes it missed. The others read some changes again.
+      const saved = states.flatMap((state) => (state?.changes ? [state.changes] : []));
+      const cursor = Arr.isReadonlyArrayNonEmpty(saved)
+        ? Arr.min(saved, bySavedAt).cursor
+        : initialCutoff;
+      const result = yield* feed.fetch({ cursor, selected }).pipe(recordFeedError("fetch"));
+
+      yield* Effect.forEach(
+        members,
+        (resource) =>
+          ingestBatch({
+            connector: connector.name,
+            resource: resource.name,
+            source: "changes",
+            batch: {
+              cursor: result.cursor,
+              rows: result.rows[resource.name] ?? [],
+            },
+          }),
+        { discard: true },
+      ).pipe(recordFeedError("ingest"));
+
+      const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
+      yield* Effect.forEach(
+        members,
+        (resource) =>
+          Effect.withSpan(
+            store
+              .setChangesState(resource.name, {
+                cursor: normalizeCursor(result.cursor),
+                lastSuccessAt,
+              })
+              .pipe(
+                Effect.andThen(
+                  Metrics.setLastSuccessTimestamp(
+                    { connector: connector.name, resource: resource.name, source: "changes" },
+                    lastSuccessAt,
+                  ),
+                ),
+                Effect.andThen(store.clearResourceError(resource.name, "changes")),
+                Effect.tapError((error) => annotateError("state_set", error)),
+              ),
+            SpanName.stateSet,
+            { attributes: { [Attr.stateKey]: resource.name } },
+          ),
+        { discard: true },
+      ).pipe(recordFeedError("checkpoint"));
+
+      yield* Effect.forEach(members, (resource, index) =>
+        states[index]?.lastError?.source === "changes"
+          ? refreshRuntimeStatus(connector, resource, initialCutoff)
+          : Effect.void,
+      );
+
+      if (!result.hasMore) {
+        yield* Effect.sleep(feed.interval ?? "1 minute");
+      }
+    }
+  }).pipe(
+    // Only typed errors park the feed. Defects and interruption still stop the runtime.
+    Effect.catch((error) =>
+      Effect.logError("Connector changes feed parked", error).pipe(
+        Effect.annotateLogs({
+          [Attr.connectorName]: connector.name,
+          resources: feed.resources.map((resource) => resource.name).join(","),
+        }),
+        Effect.andThen(Effect.never),
+      ),
+    ),
+  );

package/src/ingestor/arrow.ts

@@ -0,0 +1,248 @@
+import {
+  Bool,
+  DateDay,
+  Decimal,
+  Field,
+  FixedSizeBinary,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  LargeBinary,
+  List,
+  Map_,
+  RecordBatch,
+  Schema as ArrowSchema,
+  Struct,
+  TimeMicrosecond,
+  TimestampMicrosecond,
+  Utf8,
+  makeBuilder,
+  type DataType,
+  type TypeMap,
+} from "apache-arrow";
+import { Effect, Schema } from "effect";
+import {
+  parseDecimalType,
+  parseFixedType,
+  type IcebergType,
+  type PrimitiveType,
+  type TableSchema,
+} from "iceberg-js";
+
+import { ConnectorError } from "../errors";
+
+// Nothing converts an Iceberg schema to Arrow for us, so we do it here.
+// Each type also gets a schema for its values, because Arrow does not reject a bad
+// value, it converts it: "1" becomes 1, and 1.5 becomes 1.
+
+type RowSchema = Schema.Codec<unknown, unknown>;
+
+/** One Iceberg type: the Arrow type, and the values it accepts. */
+type Compiled = {
+  readonly type: DataType;
+  readonly row: RowSchema;
+};
+
+// Arrow pads or cuts binary values to fit.
+const bytes = (length: number) =>
+  Schema.Uint8Array.check(
+    Schema.makeFilter(
+      (value: Uint8Array) => value.byteLength === length || `expected ${length} bytes`,
+    ),
+  );
+
+// Arrow wraps big integers instead of failing, so check the range.
+const int32 = Schema.Number.check(
+  Schema.isInt(),
+  Schema.isBetween({ minimum: -2_147_483_648, maximum: 2_147_483_647 }),
+);
+
+const int64 = Schema.BigInt.check(
+  Schema.isBetweenBigInt({ minimum: -(2n ** 63n), maximum: 2n ** 63n - 1n }),
+);
+
+// Arrow turns a number outside the 32-bit range into Infinity.
+const float32 = Schema.Finite.check(
+  Schema.makeFilter(
+    (value: number) => Number.isFinite(Math.fround(value)) || "expected a 32-bit number",
+  ),
+);
+
+const microsecondsInDay = Schema.BigInt.check(
+  Schema.isBetweenBigInt({ minimum: 0n, maximum: 86_399_999_999n }),
+);
+
+const primitives: Record<string, Compiled> = {
+  boolean: { type: new Bool(), row: Schema.Boolean },
+  int: { type: new Int32(), row: int32 },
+  long: { type: new Int64(), row: int64 },
+  float: { type: new Float32(), row: float32 },
+  double: { type: new Float64(), row: Schema.Finite },
+  string: { type: new Utf8(), row: Schema.String },
+  uuid: { type: new FixedSizeBinary(16), row: bytes(16) },
+  binary: { type: new LargeBinary(), row: Schema.Uint8Array },
+  date: { type: new DateDay(), row: Schema.Date },
+  time: { type: new TimeMicrosecond(), row: microsecondsInDay },
+  timestamp: { type: new TimestampMicrosecond(), row: Schema.Date },
+  timestamptz: { type: new TimestampMicrosecond("UTC"), row: Schema.Date },
+};
+
+const primitive = (type: PrimitiveType): Compiled => {
+  const known = primitives[type];
+
+  if (known) return known;
+
+  // Nothing checks the catalog metadata, and Arrow takes any precision.
+  // Better to fail here than at write time.
+  const decimal = parseDecimalType(type);
+
+  if (decimal) {
+    if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) {
+      throw new Error(`Invalid Iceberg decimal type ${type}`);
+    }
+
+    return {
+      type: new Decimal(decimal.scale, decimal.precision),
+      row: Schema.instanceOf(Uint32Array).check(
+        Schema.makeFilter((value: Uint32Array) => value.length === 4 || "expected four words"),
+      ),
+    };
+  }
+
+  const fixed = parseFixedType(type);
+
+  if (fixed) {
+    if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);
+
+    return { type: new FixedSizeBinary(fixed.length), row: bytes(fixed.length) };
+  }
+
+  throw new Error(`Unsupported Iceberg type ${type}`);
+};
+
+/** Absent and null both become null in Arrow. */
+const optional = (row: RowSchema) => Schema.optional(Schema.NullOr(row));
+
+const fieldMetadata = (id: number): Map<string, string> =>
+  new Map([["PARQUET:field_id", String(id)]]);
+
+/** Same as `primitive`, plus struct, list and map. */
+const compileType = (type: IcebergType): Compiled => {
+  if (typeof type === "string") return primitive(type);
+
+  if (type.type === "struct") {
+    const fields = type.fields.map((field) => ({ ...field, compiled: compileType(field.type) }));
+
+    return {
+      type: new Struct(
+        fields.map(
+          (field) =>
+            new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)),
+        ),
+      ),
+      row: Schema.Struct(
+        Object.fromEntries(
+          fields.map((field) => [
+            field.name,
+            field.required ? field.compiled.row : optional(field.compiled.row),
+          ]),
+        ),
+      ),
+    };
+  }
+
+  if (type.type === "list") {
+    const element = compileType(type.element);
+    const required = type["element-required"];
+
+    return {
+      type: new List(
+        new Field("element", element.type, !required, fieldMetadata(type["element-id"])),
+      ),
+      row: Schema.Array(required ? element.row : Schema.NullishOr(element.row)),
+    };
+  }
+
+  const key = compileType(type.key);
+  const value = compileType(type.value);
+  const required = type["value-required"];
+  const entries = new Struct([
+    new Field("key", key.type, false, fieldMetadata(type["key-id"])),
+    new Field("value", value.type, !required, fieldMetadata(type["value-id"])),
+  ]);
+
+  return {
+    type: new Map_(new Field("entries", entries, false)),
+    row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row)),
+  };
+};
+
+export type RowEncoder = (
+  rows: ReadonlyArray<object>,
+) => Effect.Effect<RecordBatch<TypeMap>, ConnectorError>;
+
+export const makeRowEncoder = (
+  tableSchema: TableSchema,
+  version: string,
+): Effect.Effect<RowEncoder, ConnectorError> =>
+  Effect.try({
+    try: () => {
+      const fields = tableSchema.fields.map((field) => ({
+        id: field.id,
+        name: field.name,
+        ...compileType(field.type),
+      }));
+      const fieldNames = new Set(fields.map((field) => field.name));
+      const identifierIds = new Set(tableSchema["identifier-field-ids"]);
+      const keys = tableSchema.fields
+        .filter((field) => identifierIds.has(field.id))
+        .map((field) => field.name);
+      // Other columns may be missing: a delete only sends the key and version.
+      const required = new Set([...keys, version]);
+
+      if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);
+
+      const struct = new Struct(
+        fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))),
+      );
+      const schema = new ArrowSchema(struct.children);
+
+      const decodeRow = Schema.decodeUnknownSync(
+        Schema.Struct(
+          Object.fromEntries(
+            fields.map((field) => [
+              field.name,
+              required.has(field.name) ? field.row : optional(field.row),
+            ]),
+          ),
+        ),
+        { onExcessProperty: "error" },
+      );
+
+      // Schema says which field is wrong. This adds the row number.
+      const decodeAt = (row: object, index: number) => {
+        try {
+          return decodeRow(row) as Record<string, unknown>;
+        } catch (cause) {
+          const detail = cause instanceof Error ? cause.message : String(cause);
+          throw new Error(`row ${index}: ${detail}`, { cause });
+        }
+      };
+
+      return (rows: ReadonlyArray<object>) =>
+        Effect.try({
+          try: () => {
+            const builder = makeBuilder({ type: struct, nullValues: [null, undefined] });
+
+            for (const [index, row] of rows.entries()) {
+              builder.append(decodeAt(row, index) as Struct<TypeMap>["TValue"]);
+            }
+
+            return new RecordBatch(schema, builder.finish().flush());
+          },
+          catch: (cause) => new ConnectorError({ message: "Failed to encode rows", cause }),
+        });
+    },
+    catch: (cause) => new ConnectorError({ message: "Failed to convert Iceberg schema", cause }),
+  });

package/src/ingestor/console.ts

@@ -0,0 +1,13 @@
+import { Effect, Layer } from "effect";
+
+import { Ingestor } from "./service";
+
+/** Logs batches without sending them to Wings. */
+export const layerConsole = Layer.succeed(Ingestor)({
+  ingest: ({ resource, source, batch }) =>
+    Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(
+      Effect.annotateLogs({ rows: batch.rows.length, cursor: batch.cursor, source }),
+      // Nothing is encoded, so there are no bytes to count.
+      Effect.as({ rows: batch.rows.length, bytes: 0 }),
+    ),
+});