@useairfoil/connector-kit 0.3.0 → 0.5.0

package/src/publisher/wings.ts

@@ -1,124 +1,204 @@
-import type { PartitionValue } from "@useairfoil/wings";
 import * as Wings from "@useairfoil/wings";
 import { Effect, Layer } from "effect";
-import { ConnectorError } from "../core/errors";
-import type { ConnectorDefinition } from "../core/types";
-import { Publisher } from "./service";
+
+import type { ConnectorDefinition, ResourceDefinition } from "../core/types";
+
+import { ConnectorError } from "../errors";
+import { SpanName } from "../telemetry";
+import { Publisher, type PublishAck, type PublisherService } from "./service";
 
 type Rows = Record<string, unknown>;
 
+export type WingsTableMapping = {
+  readonly name: string;
+  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
+};
+
 export type WingsPublisherConfig = {
   readonly connector: ConnectorDefinition;
-  /** Map of entity/event name to Wings topic. */
-  readonly topics: Record<string, Wings.Cluster.Topic.Topic>;
-  /** per-stream partition value (key is entity/event name). */
-  readonly partitionValues?: Record<string, PartitionValue>;
+  readonly tables: Record<string, string | WingsTableMapping>;
 };
 
-/** Publisher entry for a single entity/event. */
 type PublisherEntry = {
-  /** Wings publisher for the entity/event. */
+  readonly resource: ResourceDefinition;
+  readonly table: Wings.Cluster.Table.Table;
   readonly publisher: Wings.WingsClient.Publisher;
-  /** Partition field name (if any). */
-  readonly partitionField?: string;
-  /** Partition value (if any). */
-  readonly partitionValue?: PartitionValue;
+  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
+  readonly keyField: Wings.Cluster.ArrowType.Field;
+  readonly versionField: Wings.Cluster.ArrowType.Field;
 };
 
+const normalizeMapping = (mapping: string | WingsTableMapping): WingsTableMapping =>
+  typeof mapping === "string" ? { name: mapping } : mapping;
+
 const buildRecordBatch = (rows: ReadonlyArray<Rows>) => {
-  // Convert JSON rows into an Arrow RecordBatch for Wings.
-  const table = Wings.tableFromJSON(Array.from(rows));
+  const table = Wings.Arrow.tableFromJSON(Array.from(rows));
   const [batch] = table.batches;
-
-  if (!batch) {
-    throw new ConnectorError({ message: "No rows to publish" });
-  }
-
-  return batch;
+  return batch
+    ? Effect.succeed(batch)
+    : Effect.fail(new ConnectorError({ message: "No rows to publish" }));
 };
 
-export const WingsPublisherLayer = (
+const validateTableMapping = (options: {
+  readonly resource: ResourceDefinition;
+  readonly table: Wings.Cluster.Table.Table;
+  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
+}) =>
+  Effect.try({
+    try: () => {
+      const keyField = Wings.TableUtils.getKeyField(options.table);
+      const versionField = Wings.TableUtils.getVersionField(options.table);
+      const partitionField = Wings.TableUtils.getPartitionField(options.table);
+
+      if (keyField.name !== options.resource.key) {
+        throw new Error(
+          `Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`,
+        );
+      }
+      if (versionField.name !== options.resource.version) {
+        throw new Error(
+          `Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`,
+        );
+      }
+      if (options.resource.partition?.required === true && !options.partitionValue) {
+        throw new Error(`Resource ${options.resource.name} requires a partition value`);
+      }
+      if (partitionField && !options.partitionValue) {
+        throw new Error(
+          `Wings table ${options.table.name} is partitioned but no partition value was provided`,
+        );
+      }
+      if (!partitionField && options.partitionValue) {
+        throw new Error(
+          `Wings table ${options.table.name} is not partitioned but a partition value was provided`,
+        );
+      }
+
+      return { keyField, versionField };
+    },
+    catch: (cause) =>
+      new ConnectorError({
+        message: `Invalid Wings table mapping for ${options.resource.name}`,
+        cause,
+      }),
+  });
+
+const accepted = (entry: PublisherEntry): PublishAck => ({
+  status: "accepted",
+  resource: entry.resource.name,
+  partition: entry.partitionValue,
+});
+
+const rejected = (entry: PublisherEntry, reason: string, rejectedRows: number): PublishAck => ({
+  status: "rejected",
+  resource: entry.resource.name,
+  reason,
+  rejectedRows,
+  partition: entry.partitionValue,
+});
+
+export const layerWings = (
   config: WingsPublisherConfig,
 ): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>
   Layer.effect(Publisher)(
     Effect.gen(function* () {
       const entries = new Map<string, PublisherEntry>();
+      const clusterClient = yield* Wings.WingsClient.clusterClient;
 
-      // create and store a wings publisher for each entity/event.
-      for (const def of [
-        ...config.connector.entities,
-        ...config.connector.events,
-      ]) {
-        const topic = config.topics[def.name];
-        if (!topic) {
+      for (const resource of config.connector.resources) {
+        const mappingInput = config.tables[resource.name];
+        if (!mappingInput) {
           return yield* Effect.fail(
-            new ConnectorError({ message: `Missing topic for ${def.name}` }),
+            new ConnectorError({ message: `Missing table for ${resource.name}` }),
           );
         }
 
-        const partitionIndex =
-          topic.partitionKey !== undefined
-            ? topic.schema.fields.findIndex(
-                (field) => field.id === topic.partitionKey,
-              )
-            : undefined;
-
-        const partitionField =
-          partitionIndex !== undefined && partitionIndex >= 0
-            ? topic.schema.fields[partitionIndex]?.name
-            : undefined;
-
-        const publisher = yield* Wings.WingsClient.publisher({
-          topic,
-          partitionValue: config.partitionValues?.[def.name],
-        }).pipe(
+        const mapping = normalizeMapping(mappingInput);
+        const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(
           Effect.mapError(
             (error) =>
-              new ConnectorError({ message: error.message, cause: error }),
+              new ConnectorError({
+                message: `Failed to resolve Wings table for ${resource.name}`,
+                cause: error,
+              }),
           ),
         );
+        const { keyField, versionField } = yield* validateTableMapping({
+          resource,
+          table,
+          partitionValue: mapping.partitionValue,
+        });
 
-        entries.set(def.name, {
+        const publisher = yield* Wings.WingsClient.publisher({
+          table,
+          partitionValue: mapping.partitionValue,
+        }).pipe(
+          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),
+        );
+
+        entries.set(resource.name, {
+          resource,
+          table,
           publisher,
-          partitionField,
-          partitionValue: config.partitionValues?.[def.name],
+          partitionValue: mapping.partitionValue,
+          keyField,
+          versionField,
         });
       }
 
-      return {
-        publish: ({ name, batch }) =>
-          Effect.gen(function* () {
-            const entry = entries.get(name);
-            if (!entry) {
-              return yield* Effect.fail(
-                new ConnectorError({ message: `Unknown stream ${name}` }),
-              );
-            }
+      const pushRows = (
+        entry: PublisherEntry,
+        operation: "upsert" | "delete",
+        rows: ReadonlyArray<Rows>,
+      ) =>
+        Effect.gen(function* () {
+          if (rows.length === 0) return accepted(entry);
+          const batch = yield* buildRecordBatch(rows);
+          return yield* entry.publisher
+            .push({ operation, batch, partitionValue: entry.partitionValue })
+            .pipe(
+              Effect.matchEffect({
+                onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),
+                onSuccess: () => Effect.succeed(accepted(entry)),
+              }),
+            );
+        });
 
-            if (batch.rows.length === 0) {
-              return { success: true };
+      const service: PublisherService = {
+        publish: Effect.fn(SpanName.publish, { kind: "producer" })(function* ({
+          resource,
+          source: _source,
+          batch,
+        }) {
+          const entry = entries.get(resource);
+          if (!entry) {
+            return yield* Effect.fail(
+              new ConnectorError({ message: `Unknown resource ${resource}` }),
+            );
+          }
+
+          if (batch.mutations.length === 0) return accepted(entry);
+
+          const upserts: Rows[] = [];
+          const deletes: Rows[] = [];
+          for (const mutation of batch.mutations) {
+            if (mutation.op === "upsert") {
+              upserts.push(mutation.row as Rows);
+            } else {
+              deletes.push({
+                [entry.keyField.name]: mutation.key,
+                [entry.versionField.name]: mutation.version,
+              });
             }
+          }
+
+          const upsertAck = yield* pushRows(entry, "upsert", upserts);
+          if (upsertAck.status === "rejected") return upsertAck;
 
-            const recordBatch = buildRecordBatch(batch.rows);
-            const result = yield* entry.publisher
-              .push({
-                batch: recordBatch,
-                partitionValue: entry.partitionValue,
-              })
-              .pipe(
-                Effect.mapError(
-                  (error) =>
-                    new ConnectorError({
-                      message: error.message,
-                      cause: error,
-                    }),
-                ),
-              );
-
-            return {
-              success: !!(result.result && result.result.$case === "accepted"),
-            };
-          }),
+          return yield* pushRows(entry, "delete", deletes);
+        }),
       };
+
+      return Publisher.of(service);
     }),
   );

package/src/runtime/context.ts

@@ -0,0 +1,15 @@
+import { Context, Layer } from "effect";
+
+import type { ConnectorDefinition } from "../core/types";
+
+export type ConnectorRuntimeContextValue = {
+  readonly connector: ConnectorDefinition;
+};
+
+export class ConnectorRuntimeContext extends Context.Service<
+  ConnectorRuntimeContext,
+  ConnectorRuntimeContextValue
+>()("@useairfoil/connector-kit/ConnectorRuntimeContext") {}
+
+export const layer = (connector: ConnectorDefinition) =>
+  Layer.succeed(ConnectorRuntimeContext)(ConnectorRuntimeContext.of({ connector }));

package/src/state-store/index.ts

@@ -0,0 +1,2 @@
+export { layerMemory } from "./memory";
+export { StateStore, type StateStoreService } from "./service";

package/src/state-store/memory.ts

@@ -0,0 +1,25 @@
+import { Effect, Layer, Ref } from "effect";
+
+import type { ResourceState } from "../core/types";
+
+import { StateStore } from "./service";
+
+/** In-memory resource state store backed by a `Ref<Map>`. Suitable for development and tests. */
+export const layerMemory: Layer.Layer<StateStore> = Layer.effect(StateStore)(
+  Effect.gen(function* () {
+    const ref = yield* Ref.make(new Map<string, ResourceState>());
+
+    return StateStore.of({
+      getResourceState: (resource) => Effect.map(Ref.get(ref), (map) => map.get(resource)),
+      setResourceState: (resource, state) =>
+        Effect.map(
+          Ref.update(ref, (map) => {
+            const next = new Map(map);
+            next.set(resource, state);
+            return next;
+          }),
+          () => undefined,
+        ),
+    });
+  }),
+);

package/src/state-store/service.ts

@@ -0,0 +1,18 @@
+import { Context, type Effect } from "effect";
+
+import type { ResourceState } from "../core/types";
+import type { ConnectorError } from "../errors";
+
+export interface StateStoreService {
+  readonly getResourceState: (
+    resource: string,
+  ) => Effect.Effect<ResourceState | undefined, ConnectorError>;
+  readonly setResourceState: (
+    resource: string,
+    state: ResourceState,
+  ) => Effect.Effect<void, ConnectorError>;
+}
+
+export class StateStore extends Context.Service<StateStore, StateStoreService>()(
+  "@useairfoil/connector-kit/StateStore",
+) {}

package/src/telemetry.ts

@@ -0,0 +1,246 @@
+import { Cause, Config, Effect, Layer, Option } from "effect";
+import { Headers } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
+
+/** Canonical span names emitted by connector-kit runtime instrumentation. */
+export const SpanName = {
+  batchProcess: "connector.batch.process",
+  webhookDecode: "connector.webhook.decode",
+  webhookHandle: "connector.webhook.handle",
+  publish: "connector.publish",
+  stateSet: "connector.state.set",
+  apiFetch: "connector.api.fetch",
+} as const;
+
+/** Canonical span attribute names. Keep these stable for collector queries. */
+export const Attr = {
+  connectorName: "airfoil.connector.name",
+  connectorResourcesCount: "airfoil.connector.resources.count",
+  resourceName: "airfoil.resource.name",
+  resourceSource: "airfoil.resource.source",
+  batchMutations: "airfoil.batch.mutations",
+  webhookPath: "airfoil.webhook.path",
+  publisherSuccess: "airfoil.publisher.success",
+  stateKey: "airfoil.state.key",
+  apiPath: "airfoil.api.path",
+  errorPhase: "airfoil.error.phase",
+  errorType: "airfoil.error.type",
+  errorMessage: "airfoil.error.message",
+  errorDetails: "airfoil.error.details",
+} as const;
+
+/** Span event names for high-cardinality reproduction details. */
+export const EventName = {
+  batchCheckpoint: "airfoil.batch.checkpoint",
+} as const;
+
+/** Event-only attributes. These are intentionally not span attributes. */
+export const EventAttr = {
+  batchCursor: "airfoil.batch.cursor",
+} as const;
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null;
+
+const maxErrorMessageLength = 500;
+
+const OtlpEnvConfig = Config.all({
+  enabled: Config.boolean("OTEL_ENABLED").pipe(Config.withDefault(false)),
+  baseUrl: Config.option(Config.string("OTEL_EXPORTER_OTLP_ENDPOINT")),
+  rawHeaders: Config.option(Config.string("OTEL_EXPORTER_OTLP_HEADERS")),
+});
+
+const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
+  "authorization",
+  "cookie",
+  "set-cookie",
+  "x-api-key",
+  /api[-_]?key/i,
+  /secret/i,
+  /signature/i,
+  /token/i,
+];
+
+const parseOtelHeaders = (value: string): Record<string, string> =>
+  Object.fromEntries(
+    value
+      .split(",")
+      .map((entry) => entry.trim())
+      .filter(Boolean)
+      .flatMap((entry) => {
+        const separator = entry.indexOf("=");
+        if (separator < 1) return [];
+        return [[entry.slice(0, separator).trim(), entry.slice(separator + 1).trim()]];
+      }),
+  );
+
+const appendPath = (baseUrl: string, path: string) =>
+  `${baseUrl.replace(/\/$/, "")}/${path.replace(/^\//, "")}`;
+
+/**
+ * Best-effort stable error classifier for span metadata.
+ *
+ * Effect/Data tagged errors expose `_tag`; that is the most useful value for
+ * connector bugs (`ConnectorError`, `InvalidWebhookPayloadError`, etc.). For
+ * non-tagged errors we fall back to the JS `Error.name`, then constructor name,
+ * then the primitive type.
+ */
+const errorType = (error: unknown) => {
+  if (isRecord(error) && typeof error._tag === "string") {
+    return String(error._tag);
+  }
+  if (error instanceof Error) {
+    return error.name;
+  }
+  if (isRecord(error) && typeof error.constructor?.name === "string") {
+    return error.constructor.name;
+  }
+  return typeof error;
+};
+
+/**
+ * Human-readable error summary for span metadata.
+ *
+ * Tagged errors in this package carry a `message` field, so this preserves the
+ * important diagnostic text. Unknown object failures are intentionally reduced
+ * to their type; full structured error payloads belong in logs, not span attrs.
+ */
+const errorMessage = (error: unknown) => {
+  const message = (() => {
+    if (isRecord(error) && typeof error.message === "string") {
+      return error.message;
+    }
+    if (typeof error === "string") {
+      return error;
+    }
+    return errorType(error);
+  })();
+
+  if (message.length > maxErrorMessageLength) {
+    return `${message.slice(0, maxErrorMessageLength)}...`;
+  }
+
+  return message;
+};
+
+const errorDetails = (error: unknown): string | undefined => {
+  if (!isRecord(error) || error.cause === undefined) return undefined;
+  return Cause.pretty(Cause.die(error.cause));
+};
+
+/** Annotates the active span with phase-aware error metadata before re-failing. */
+export const annotateError = (phase: string, error: unknown) => {
+  const details = errorDetails(error);
+  return Effect.annotateCurrentSpan({
+    [Attr.errorPhase]: phase,
+    [Attr.errorType]: errorType(error),
+    [Attr.errorMessage]: errorMessage(error),
+    ...(details !== undefined ? { [Attr.errorDetails]: details } : {}),
+  });
+};
+
+/** Adds an event to the active span, or no-ops when no span is active. */
+export const addCurrentSpanEvent = (name: string, attributes?: Record<string, unknown>) =>
+  Effect.currentSpan.pipe(
+    Effect.flatMap((span) =>
+      Effect.clockWith((clock) =>
+        Effect.sync(() => span.event(name, clock.currentTimeNanosUnsafe(), attributes)),
+      ),
+    ),
+    Effect.ignore,
+  );
+
+/** Transport configuration for OTLP trace export. All fields are direct values. */
+export type OtlpTracingConfig = {
+  readonly enabled?: boolean;
+  readonly endpoint?: string;
+  readonly headers?: Record<string, string>;
+};
+
+/** Runtime options — not config-wrappable since RegExp cannot live in Config. */
+export type OtlpTracingOptions = {
+  /** Additional connector-specific header names or regexes to redact from HTTP logs/traces. */
+  readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
+};
+
+const buildOtlpTracingLayer = (config: OtlpTracingConfig, options: OtlpTracingOptions) =>
+  Effect.gen(function* () {
+    const allRedacted = [...defaultRedactedHeaders, ...(options.redactedHeaders ?? [])];
+    const RedactionLayer = Layer.succeed(Headers.CurrentRedactedNames)(allRedacted);
+
+    if (!config.enabled) {
+      return RedactionLayer;
+    }
+
+    if (!config.endpoint) {
+      return yield* Effect.fail(new Error("enabled=true requires endpoint"));
+    }
+
+    yield* Effect.logInfo("telemetry enabled").pipe(
+      Effect.annotateLogs({
+        endpoint: config.endpoint,
+        headers: config.headers ? Object.keys(config.headers) : [],
+      }),
+    );
+
+    return Layer.mergeAll(
+      RedactionLayer,
+      Observability.OtlpTracer.layer({
+        url: appendPath(config.endpoint, "/v1/traces"),
+        headers: config.headers,
+      }).pipe(Layer.provide(Observability.OtlpSerialization.layerJson)),
+    );
+  });
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Takes direct config values — no `ConfigProvider` required.
+ */
+export const layer = (config: OtlpTracingConfig = {}, options: OtlpTracingOptions = {}) =>
+  Layer.unwrap(buildOtlpTracingLayer(config, options));
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Takes Effect Config-wrapped values — reads from `ConfigProvider`.
+ *
+ * @example
+ * Telemetry.layerConfig({
+ *   enabled: Config.boolean("MY_OTEL_ENABLED"),
+ *   endpoint: Config.string("MY_COLLECTOR_URL"),
+ * })
+ */
+export const layerConfig = (
+  config: Config.Wrap<OtlpTracingConfig>,
+  options: OtlpTracingOptions = {},
+) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const resolved = yield* Config.unwrap(config);
+      return yield* buildOtlpTracingLayer(resolved, options);
+    }),
+  );
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Zero-config shortcut that reads the standard `OTEL_*` env var names.
+ *
+ * Reads `OTEL_ENABLED`, `OTEL_EXPORTER_OTLP_ENDPOINT`, and
+ * `OTEL_EXPORTER_OTLP_HEADERS`. Effect's OTLP resource handling reads
+ * `OTEL_SERVICE_NAME`, `OTEL_SERVICE_VERSION`, and `OTEL_RESOURCE_ATTRIBUTES`.
+ */
+export const layerOtlpTracing = (options: OtlpTracingOptions = {}) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const raw = yield* OtlpEnvConfig;
+      return yield* buildOtlpTracingLayer(
+        {
+          enabled: raw.enabled,
+          endpoint: Option.getOrUndefined(raw.baseUrl),
+          headers: Option.isSome(raw.rawHeaders)
+            ? parseOtelHeaders(raw.rawHeaders.value)
+            : undefined,
+        },
+        options,
+      );
+    }),
+  );

package/src/webhook/index.ts

@@ -0,0 +1,8 @@
+import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from "../core/types";
+
+export { router } from "./server";
+export type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from "./types";
+
+export const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(
+  definition: WebhookRouteInput<Resources, Payload>,
+): WebhookRoute<Resources, Payload> => definition;