@useairfoil/connector-kit 0.6.0 → 0.7.0

package/src/ingestor/arrow.ts

@@ -0,0 +1,248 @@
+import {
+  Bool,
+  DateDay,
+  Decimal,
+  Field,
+  FixedSizeBinary,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  LargeBinary,
+  List,
+  Map_,
+  RecordBatch,
+  Schema as ArrowSchema,
+  Struct,
+  TimeMicrosecond,
+  TimestampMicrosecond,
+  Utf8,
+  makeBuilder,
+  type DataType,
+  type TypeMap,
+} from "apache-arrow";
+import { Effect, Schema } from "effect";
+import {
+  parseDecimalType,
+  parseFixedType,
+  type IcebergType,
+  type PrimitiveType,
+  type TableSchema,
+} from "iceberg-js";
+
+import { ConnectorError } from "../errors";
+
+// Nothing converts an Iceberg schema to Arrow for us, so we do it here.
+// Each type also gets a schema for its values, because Arrow does not reject a bad
+// value, it converts it: "1" becomes 1, and 1.5 becomes 1.
+
+type RowSchema = Schema.Codec<unknown, unknown>;
+
+/** One Iceberg type: the Arrow type, and the values it accepts. */
+type Compiled = {
+  readonly type: DataType;
+  readonly row: RowSchema;
+};
+
+// Arrow pads or cuts binary values to fit.
+const bytes = (length: number) =>
+  Schema.Uint8Array.check(
+    Schema.makeFilter(
+      (value: Uint8Array) => value.byteLength === length || `expected ${length} bytes`,
+    ),
+  );
+
+// Arrow wraps big integers instead of failing, so check the range.
+const int32 = Schema.Number.check(
+  Schema.isInt(),
+  Schema.isBetween({ minimum: -2_147_483_648, maximum: 2_147_483_647 }),
+);
+
+const int64 = Schema.BigInt.check(
+  Schema.isBetweenBigInt({ minimum: -(2n ** 63n), maximum: 2n ** 63n - 1n }),
+);
+
+// Arrow turns a number outside the 32-bit range into Infinity.
+const float32 = Schema.Finite.check(
+  Schema.makeFilter(
+    (value: number) => Number.isFinite(Math.fround(value)) || "expected a 32-bit number",
+  ),
+);
+
+const microsecondsInDay = Schema.BigInt.check(
+  Schema.isBetweenBigInt({ minimum: 0n, maximum: 86_399_999_999n }),
+);
+
+const primitives: Record<string, Compiled> = {
+  boolean: { type: new Bool(), row: Schema.Boolean },
+  int: { type: new Int32(), row: int32 },
+  long: { type: new Int64(), row: int64 },
+  float: { type: new Float32(), row: float32 },
+  double: { type: new Float64(), row: Schema.Finite },
+  string: { type: new Utf8(), row: Schema.String },
+  uuid: { type: new FixedSizeBinary(16), row: bytes(16) },
+  binary: { type: new LargeBinary(), row: Schema.Uint8Array },
+  date: { type: new DateDay(), row: Schema.Date },
+  time: { type: new TimeMicrosecond(), row: microsecondsInDay },
+  timestamp: { type: new TimestampMicrosecond(), row: Schema.Date },
+  timestamptz: { type: new TimestampMicrosecond("UTC"), row: Schema.Date },
+};
+
+const primitive = (type: PrimitiveType): Compiled => {
+  const known = primitives[type];
+
+  if (known) return known;
+
+  // Nothing checks the catalog metadata, and Arrow takes any precision.
+  // Better to fail here than at write time.
+  const decimal = parseDecimalType(type);
+
+  if (decimal) {
+    if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) {
+      throw new Error(`Invalid Iceberg decimal type ${type}`);
+    }
+
+    return {
+      type: new Decimal(decimal.scale, decimal.precision),
+      row: Schema.instanceOf(Uint32Array).check(
+        Schema.makeFilter((value: Uint32Array) => value.length === 4 || "expected four words"),
+      ),
+    };
+  }
+
+  const fixed = parseFixedType(type);
+
+  if (fixed) {
+    if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);
+
+    return { type: new FixedSizeBinary(fixed.length), row: bytes(fixed.length) };
+  }
+
+  throw new Error(`Unsupported Iceberg type ${type}`);
+};
+
+/** Absent and null both become null in Arrow. */
+const optional = (row: RowSchema) => Schema.optional(Schema.NullOr(row));
+
+const fieldMetadata = (id: number): Map<string, string> =>
+  new Map([["PARQUET:field_id", String(id)]]);
+
+/** Same as `primitive`, plus struct, list and map. */
+const compileType = (type: IcebergType): Compiled => {
+  if (typeof type === "string") return primitive(type);
+
+  if (type.type === "struct") {
+    const fields = type.fields.map((field) => ({ ...field, compiled: compileType(field.type) }));
+
+    return {
+      type: new Struct(
+        fields.map(
+          (field) =>
+            new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)),
+        ),
+      ),
+      row: Schema.Struct(
+        Object.fromEntries(
+          fields.map((field) => [
+            field.name,
+            field.required ? field.compiled.row : optional(field.compiled.row),
+          ]),
+        ),
+      ),
+    };
+  }
+
+  if (type.type === "list") {
+    const element = compileType(type.element);
+    const required = type["element-required"];
+
+    return {
+      type: new List(
+        new Field("element", element.type, !required, fieldMetadata(type["element-id"])),
+      ),
+      row: Schema.Array(required ? element.row : Schema.NullishOr(element.row)),
+    };
+  }
+
+  const key = compileType(type.key);
+  const value = compileType(type.value);
+  const required = type["value-required"];
+  const entries = new Struct([
+    new Field("key", key.type, false, fieldMetadata(type["key-id"])),
+    new Field("value", value.type, !required, fieldMetadata(type["value-id"])),
+  ]);
+
+  return {
+    type: new Map_(new Field("entries", entries, false)),
+    row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row)),
+  };
+};
+
+export type RowEncoder = (
+  rows: ReadonlyArray<object>,
+) => Effect.Effect<RecordBatch<TypeMap>, ConnectorError>;
+
+export const makeRowEncoder = (
+  tableSchema: TableSchema,
+  version: string,
+): Effect.Effect<RowEncoder, ConnectorError> =>
+  Effect.try({
+    try: () => {
+      const fields = tableSchema.fields.map((field) => ({
+        id: field.id,
+        name: field.name,
+        ...compileType(field.type),
+      }));
+      const fieldNames = new Set(fields.map((field) => field.name));
+      const identifierIds = new Set(tableSchema["identifier-field-ids"]);
+      const keys = tableSchema.fields
+        .filter((field) => identifierIds.has(field.id))
+        .map((field) => field.name);
+      // Other columns may be missing: a delete only sends the key and version.
+      const required = new Set([...keys, version]);
+
+      if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);
+
+      const struct = new Struct(
+        fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))),
+      );
+      const schema = new ArrowSchema(struct.children);
+
+      const decodeRow = Schema.decodeUnknownSync(
+        Schema.Struct(
+          Object.fromEntries(
+            fields.map((field) => [
+              field.name,
+              required.has(field.name) ? field.row : optional(field.row),
+            ]),
+          ),
+        ),
+        { onExcessProperty: "error" },
+      );
+
+      // Schema says which field is wrong. This adds the row number.
+      const decodeAt = (row: object, index: number) => {
+        try {
+          return decodeRow(row) as Record<string, unknown>;
+        } catch (cause) {
+          const detail = cause instanceof Error ? cause.message : String(cause);
+          throw new Error(`row ${index}: ${detail}`, { cause });
+        }
+      };
+
+      return (rows: ReadonlyArray<object>) =>
+        Effect.try({
+          try: () => {
+            const builder = makeBuilder({ type: struct, nullValues: [null, undefined] });
+
+            for (const [index, row] of rows.entries()) {
+              builder.append(decodeAt(row, index) as Struct<TypeMap>["TValue"]);
+            }
+
+            return new RecordBatch(schema, builder.finish().flush());
+          },
+          catch: (cause) => new ConnectorError({ message: "Failed to encode rows", cause }),
+        });
+    },
+    catch: (cause) => new ConnectorError({ message: "Failed to convert Iceberg schema", cause }),
+  });

package/src/ingestor/console.ts

@@ -0,0 +1,13 @@
+import { Effect, Layer } from "effect";
+
+import { Ingestor } from "./service";
+
+/** Logs batches without sending them over Arrow Flight. */
+export const layerConsole = Layer.succeed(Ingestor)({
+  ingest: ({ resource, source, batch }) =>
+    Effect.logInfo(`[ingestor] -> Source: ${source} | Resource: ${resource}`).pipe(
+      Effect.annotateLogs({ rows: batch.rows.length, cursor: batch.cursor, source }),
+      // Nothing is encoded, so there are no bytes to count.
+      Effect.as({ rows: batch.rows.length, bytes: 0 }),
+    ),
+});

package/src/ingestor/index.ts

@@ -0,0 +1,13 @@
+export { layerConsole } from "./console";
+export {
+  Ingestor,
+  type IngestOptions,
+  type IngestorService,
+  type IngestResult,
+  type IngestSource,
+} from "./service";
+export {
+  layerArrowFlight,
+  layerArrowFlightConfig,
+  type ArrowFlightIngestorConfig,
+} from "./arrow-flight";

package/src/ingestor/instrumented.ts

@@ -0,0 +1,61 @@
+import { Effect, Metric } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+
+import * as Metrics from "../metrics";
+import { Attr, EventAttr, EventName, SpanName, addCurrentSpanEvent } from "../telemetry";
+import { Ingestor, type IngestSource } from "./service";
+
+export const ingestBatch = Effect.fnUntraced(function* (options: {
+  readonly connector?: string;
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+}) {
+  const metricAttributes = Metrics.withResourceAttributes({
+    connector: options.connector ?? "unknown",
+    resource: options.resource,
+    source: options.source,
+  });
+  const batchOutcome = (outcome: "success" | "error") =>
+    Metric.withAttributes(Metrics.batches, {
+      ...metricAttributes,
+      [Attr.batchOutcome]: outcome,
+    });
+
+  const ingestor = yield* Ingestor;
+
+  const ingest = ingestor.ingest(options).pipe(
+    Effect.tap(() => Effect.annotateCurrentSpan({ [Attr.ingestionSuccess]: true })),
+    Effect.tapError(() =>
+      Effect.annotateCurrentSpan({
+        [Attr.ingestionSuccess]: false,
+        [Attr.errorPhase]: "ingest",
+      }),
+    ),
+  );
+
+  // onError keeps the failure, so the engine still sees it and skips the checkpoint.
+  const result = yield* Effect.withSpan(ingest, SpanName.ingest, { kind: "producer" }).pipe(
+    Effect.trackDuration(Metric.withAttributes(Metrics.ingestDuration, metricAttributes)),
+    Effect.onError(() => Metric.update(batchOutcome("error"), 1)),
+  );
+
+  yield* Metric.update(batchOutcome("success"), 1);
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.rowsIngested, metricAttributes),
+    options.batch.rows.length,
+  );
+  yield* Metric.update(
+    Metric.withAttributes(Metrics.batchSize, metricAttributes),
+    options.batch.rows.length,
+  );
+
+  if (options.batch.cursor !== undefined) {
+    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
+      [EventAttr.batchCursor]: options.batch.cursor,
+    });
+  }
+
+  return result;
+});

package/src/ingestor/service.ts

@@ -0,0 +1,30 @@
+import { Context, type Effect } from "effect";
+
+import type { ResourceBatch } from "../core/types";
+import type { ConnectorError } from "../errors";
+
+export type IngestSource = "backfill" | "changes" | "webhook" | "demo";
+
+/** A resource batch and the source that produced it. */
+export type IngestOptions = {
+  readonly resource: string;
+  readonly source: IngestSource;
+  readonly batch: ResourceBatch;
+};
+
+/** Rows written, and their size as uncompressed Arrow buffers. */
+export type IngestResult = {
+  readonly rows: number;
+  readonly bytes: number;
+};
+
+/** Handles batches from connector sources. */
+export interface IngestorService {
+  readonly ingest: (options: IngestOptions) => Effect.Effect<IngestResult, ConnectorError>;
+  /** Resources it has tables for. Only these run. Without it, every resource runs. */
+  readonly resourceNames?: ReadonlyArray<string>;
+}
+
+export class Ingestor extends Context.Service<Ingestor, IngestorService>()(
+  "@useairfoil/connector-kit/Ingestor",
+) {}

package/src/manifest/config.ts

@@ -111,7 +111,7 @@ const withDefault = <A extends string | number | boolean>(
 
 /** Defines a required non-empty string, optionally with a default. */
 export const string = (options: FieldOptions<string>): ConfigField<string> => ({
-  config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
+  config: withDefault(Config.NonEmptyString(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "string",
@@ -145,7 +145,7 @@ export const number = (options: NumberFieldOptions): ConfigField<number> => {
 
 /** Defines a boolean, optionally with a default. */
 export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({
-  config: withDefault(Config.boolean(options.runtimeKey), options.default),
+  config: withDefault(Config.Boolean(options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "boolean",
@@ -160,7 +160,7 @@ export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> =>
 export const select = <const Values extends readonly [string, ...Array<string>]>(
   options: FieldOptions<Values[number]> & { readonly values: Values },
 ): ConfigField<Values[number]> => ({
-  config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
+  config: withDefault(Config.Literals(options.values, options.runtimeKey), options.default),
   spec: {
     runtimeKey: options.runtimeKey,
     type: "select",

package/src/metrics.ts

@@ -1,11 +1,21 @@
-import { DateTime, Effect, Metric, Option } from "effect";
+import {
+  Cause,
+  DateTime,
+  type Duration,
+  Effect,
+  Metric,
+  Option,
+  Predicate,
+  Schedule,
+} from "effect";
+import { HttpClientError, HttpClientResponse } from "effect/http";
 
 import type { SyncState } from "./core/types";
 
 import { Attr } from "./telemetry";
 
-export type BatchSource = "backfill" | "changes" | "webhook";
-export type BatchOutcome = "accepted" | "rejected" | "error";
+export type BatchSource = "backfill" | "changes" | "webhook" | "demo";
+export type BatchOutcome = "success" | "error";
 export type ApiRetryReason = "transport" | "timeout" | "rate_limit" | "server_error";
 export type WebhookOutcome =
   | "ok"
@@ -43,32 +53,26 @@ export type ApiRetryMetricAttrs = {
   readonly reason: ApiRetryReason;
 };
 
-export const entitiesUpserted = Metric.counter("airfoil.connector.entity.upserts", {
-  description: "Total entity upsert mutations published by connector sources",
-  incremental: true,
-  attributes: { unit: "{mutation}" },
-});
-
-export const entitiesDeleted = Metric.counter("airfoil.connector.entity.deletes", {
-  description: "Total entity delete mutations published by connector sources",
+export const batches = Metric.counter("airfoil.connector.batches", {
+  description: "Total resource batches ingested by connector sources",
   incremental: true,
-  attributes: { unit: "{mutation}" },
+  attributes: { unit: "{batch}" },
 });
 
-export const batches = Metric.counter("airfoil.connector.batches", {
-  description: "Total resource batches published by connector sources",
+export const rowsIngested = Metric.counter("airfoil.connector.rows.ingested", {
+  description: "Total resource rows successfully ingested",
   incremental: true,
-  attributes: { unit: "{batch}" },
+  attributes: { unit: "{row}" },
 });
 
 export const batchSize = Metric.histogram("airfoil.connector.batch.size", {
-  description: "Distribution of resource mutation batch sizes",
+  description: "Distribution of resource row batch sizes",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],
-  attributes: { unit: "{mutation}" },
+  attributes: { unit: "{row}" },
 });
 
-export const publishDuration = Metric.timer("airfoil.connector.publish.duration", {
-  description: "Publisher call duration in milliseconds",
+export const ingestDuration = Metric.timer("airfoil.connector.ingest.duration", {
+  description: "Ingestor call duration in milliseconds",
   boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
   attributes: { unit: "ms" },
 });
@@ -80,7 +84,7 @@ export const webhookRequests = Metric.counter("airfoil.connector.webhook.request
 });
 
 export const webhookQueueDepth = Metric.gauge("airfoil.connector.webhook.queue.depth", {
-  description: "Current queued webhook batches waiting for publish",
+  description: "Current queued webhook batches waiting for ingestion",
   attributes: { unit: "{batch}" },
 });
 
@@ -154,6 +158,48 @@ export const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>
     1,
   );
 
+// Same statuses as `HttpClient.retryTransient`.
+const transientStatuses = new Set([408, 429, 500, 502, 503, 504]);
+
+// A connector only retries a `403` when the provider uses it for rate limits.
+const reasonForStatus = (status: number): ApiRetryReason =>
+  status === 408 ? "timeout" : status === 429 || status === 403 ? "rate_limit" : "server_error";
+
+// Effect exports the response type ID, not a guard.
+const isResponse = (value: unknown): value is HttpClientResponse.HttpClientResponse =>
+  Predicate.hasProperty(value, HttpClientResponse.TypeId);
+
+/** Why `HttpClient.retryTransient` retries. Undefined for a response it keeps. */
+const retryReason = (value: unknown): ApiRetryReason | undefined => {
+  if (isResponse(value)) {
+    return transientStatuses.has(value.status) ? reasonForStatus(value.status) : undefined;
+  }
+  if (HttpClientError.isHttpClientError(value) && value.reason._tag === "StatusCodeError") {
+    return reasonForStatus(value.reason.response.status);
+  }
+  return Cause.isTimeoutError(value) ? "timeout" : "transport";
+};
+
+/**
+ * A jittered exponential schedule for `HttpClient.retryTransient` that records
+ * each retry with its reason.
+ */
+export const retrySchedule = (options: {
+  readonly connector: string;
+  readonly baseDelay: Duration.Input;
+  readonly times: number;
+}) =>
+  Schedule.exponential(options.baseDelay).pipe(
+    Schedule.jittered,
+    Schedule.upTo({ times: options.times }),
+    Schedule.tap(({ input }) => {
+      const reason = retryReason(input);
+      return reason === undefined
+        ? Effect.void
+        : recordApiRetry({ connector: options.connector, reason });
+    }),
+  );
+
 const syncStates = ["pending", "backfilling", "live", "error"] as const;
 
 export const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>

package/src/runtime-config/config.ts

@@ -1,16 +1,19 @@
-import type { WingsClientOptions } from "@useairfoil/wings/wings-client";
-
 import { Config } from "effect";
 
 import { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 
 /** Shared HTTP port used by hosted and sandbox connector entrypoints. */
-export const httpPort = Config.port(PlatformRuntimeKey.httpPort).pipe(
+export const httpPort = Config.Port(PlatformRuntimeKey.httpPort).pipe(
   Config.withDefault(PlatformRuntimeDefault.httpPort),
 );
 
-/** Shared Wings client configuration supplied by the hosting platform. */
-export const wingsClient = {
-  host: Config.nonEmptyString(PlatformRuntimeKey.wingsHost),
-  namespace: Config.nonEmptyString(PlatformRuntimeKey.wingsNamespace),
-} satisfies Config.Wrap<WingsClientOptions>;
+/** Engine URL, including the protocol. */
+export const engineUri = Config.URL(PlatformRuntimeKey.engineUri);
+
+/** Arrow Flight URL. Uses the engine URL when unset. */
+export const flightUri = engineUri.pipe(
+  Config.flatMap((uri) => Config.URL(PlatformRuntimeKey.flightUri).pipe(Config.withDefault(uri))),
+);
+
+/** Catalog ID. */
+export const catalog = Config.NonEmptyString(PlatformRuntimeKey.catalog);

package/src/runtime-config/constants.ts

@@ -8,8 +8,9 @@ export const PlatformRuntimeKey = {
   connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
   stateTable: "AIRFOIL_STATE_TABLE",
   postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-  wingsHost: "WINGS_HOST",
-  wingsNamespace: "WINGS_NAMESPACE",
+  engineUri: "ENGINE_URI",
+  flightUri: "FLIGHT_URI",
+  catalog: "AIRFOIL_CATALOG",
   otelEnabled: "OTEL_ENABLED",
   otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
   otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",

package/src/runtime-config/index.ts

@@ -1,10 +1,5 @@
 export { RuntimeConfigError, type RuntimeConfigErrorCode } from "./error";
-export { httpPort, wingsClient } from "./config";
+export { catalog, engineUri, flightUri, httpPort } from "./config";
 export { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
 export { layerHosted } from "./layer";
-export {
-  type HostedTableBinding,
-  type ResolvedTableBinding,
-  type ResolvedTableBindings,
-  loadTableBindings,
-} from "./table-bindings";
+export { type ResolvedTableBindings, type TableBinding, loadTableBindings } from "./table-bindings";

package/src/runtime-config/layer.ts

@@ -8,7 +8,7 @@ const providerFromFile: Effect.Effect<
   RuntimeConfigError,
   FileSystem.FileSystem
 > = Effect.gen(function* () {
-  const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(
+  const configPath = yield* Config.NonEmptyString(PlatformRuntimeKey.configPath).pipe(
     Effect.mapError(
       (error) =>
         new RuntimeConfigError({