@useairfoil/connector-kit 0.4.0 → 0.5.1

package/src/state-store/schema.ts

@@ -0,0 +1,57 @@
+import { Schema } from "effect";
+
+const CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);
+
+/** Durable progress for a finite historical backfill. */
+export const BackfillStateSchema = Schema.Struct({
+  cutoff: CursorValueSchema,
+  pageCursor: Schema.optional(CursorValueSchema),
+  completed: Schema.Boolean,
+  lastSuccessAt: Schema.optional(Schema.String),
+});
+
+/** Durable cursor for an ongoing incremental changes stream. */
+export const ChangesStateSchema = Schema.Struct({
+  cursor: CursorValueSchema,
+  lastSuccessAt: Schema.optional(Schema.String),
+});
+
+export const StateSourceSchema = Schema.Literals(["backfill", "changes"]);
+
+export const StateOperationSchema = Schema.Literals(["fetch", "publish", "checkpoint"]);
+
+export const StateErrorCodeSchema = Schema.Literals([
+  "fetch_failed",
+  "publish_failed",
+  "checkpoint_failed",
+]);
+
+/** SDK-owned error context safe to persist and expose through status. */
+export const ResourceErrorContextSchema = Schema.Struct({
+  source: StateSourceSchema,
+  operation: StateOperationSchema,
+  code: StateErrorCodeSchema,
+  message: Schema.String,
+});
+
+/** Stable public error summary persisted for status reporting; never a raw cause. */
+export const ResourceErrorSchema = Schema.Struct({
+  ...ResourceErrorContextSchema.fields,
+  at: Schema.String,
+});
+
+/** Complete persisted state assembled from independently stored components. */
+export const ResourceStateSchema = Schema.Struct({
+  backfill: Schema.optional(BackfillStateSchema),
+  changes: Schema.optional(ChangesStateSchema),
+  lastError: Schema.optional(ResourceErrorSchema),
+});
+
+export type PersistedBackfillState = Schema.Schema.Type<typeof BackfillStateSchema>;
+export type PersistedChangesState = Schema.Schema.Type<typeof ChangesStateSchema>;
+export type StateSource = Schema.Schema.Type<typeof StateSourceSchema>;
+export type StateOperation = Schema.Schema.Type<typeof StateOperationSchema>;
+export type StateErrorCode = Schema.Schema.Type<typeof StateErrorCodeSchema>;
+export type ResourceErrorContext = Schema.Schema.Type<typeof ResourceErrorContextSchema>;
+export type PersistedResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;
+export type PersistedResourceState = Schema.Schema.Type<typeof ResourceStateSchema>;

package/src/state-store/service.ts

@@ -0,0 +1,51 @@
+import { Context, Effect } from "effect";
+
+import type {
+  PersistedBackfillState,
+  PersistedChangesState,
+  PersistedResourceState,
+  StateOperation,
+  StateSource,
+} from "./schema";
+
+import { ConnectorError } from "../errors";
+
+/** Connector-facing durable state operations, independent of the storage backend. */
+export interface StateStoreService {
+  readonly getResourceState: (
+    resource: string,
+  ) => Effect.Effect<PersistedResourceState | undefined, ConnectorError>;
+  readonly setBackfillState: (
+    resource: string,
+    state: PersistedBackfillState,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly setChangesState: (
+    resource: string,
+    state: PersistedChangesState,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly setResourceError: (
+    resource: string,
+    source: StateSource,
+    operation: StateOperation,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly clearResourceError: (
+    resource: string,
+    source: StateSource,
+  ) => Effect.Effect<void, ConnectorError>;
+}
+
+/**
+ * Effect service used by ingestion and status logic for connector checkpoints.
+ * Production provides `layerSql`; sandboxes and tests may provide `layerMemory`.
+ *
+ * @example
+ * ```ts
+ * const program = Effect.gen(function* () {
+ *   const state = yield* StateStore;
+ *   yield* state.setChangesState("orders", { cursor: "next-page" });
+ * });
+ * ```
+ */
+export class StateStore extends Context.Service<StateStore, StateStoreService>()(
+  "@useairfoil/connector-kit/StateStore",
+) {}

package/src/state-store/state.ts

@@ -0,0 +1,17 @@
+import type { Cursor, ResourceDefinition, ResourceState, SyncState } from "../core/types";
+
+/** Normalizes a cursor value to its persisted/wire representation (never a Date). */
+export const normalizeCursor = (value: Cursor.Value): string | number =>
+  value instanceof Date ? value.toISOString() : value;
+
+/** Derives connector-facing sync state from durable state and resource capabilities. */
+export const deriveSyncState = (
+  resource: ResourceDefinition,
+  state: ResourceState | undefined,
+): SyncState => {
+  if (state?.lastError) return "error";
+  if (!resource.backfill && !resource.changes) return "live";
+  if (!state) return "pending";
+  if (resource.backfill && (!state.backfill || !state.backfill.completed)) return "backfilling";
+  return "live";
+};

package/src/status/index.ts

@@ -0,0 +1,2 @@
+export * from "./route";
+export * from "./types";

package/src/status/route.ts

@@ -0,0 +1,77 @@
+import { Cause, Effect } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+
+import type { ConnectorDefinition } from "../core/types";
+import type { ResourceStatus, StatusUnavailableResponse } from "./types";
+
+import { StateStore } from "../state-store";
+import { deriveSyncState, normalizeCursor } from "../state-store/state";
+
+export const statusRoute = (
+  connector: ConnectorDefinition,
+  path: HttpRouter.PathInput = "/status",
+) =>
+  HttpRouter.add(
+    "GET",
+    path,
+    Effect.gen(function* () {
+      const store = yield* StateStore;
+
+      const resources = yield* Effect.forEach(
+        connector.resources,
+        (resource) =>
+          store.getResourceState(resource.name).pipe(
+            Effect.map(
+              (state): ResourceStatus => ({
+                name: resource.name,
+                state: deriveSyncState(resource, state),
+                ...(state?.backfill
+                  ? {
+                      backfill: {
+                        completed: state.backfill.completed,
+                        cutoff: normalizeCursor(state.backfill.cutoff),
+                        ...(state.backfill.pageCursor !== undefined
+                          ? { pageCursor: normalizeCursor(state.backfill.pageCursor) }
+                          : {}),
+                        ...(state.backfill.lastSuccessAt !== undefined
+                          ? { lastSuccessAt: state.backfill.lastSuccessAt }
+                          : {}),
+                      },
+                    }
+                  : {}),
+                ...(state?.changes
+                  ? {
+                      changes: {
+                        cursor: normalizeCursor(state.changes.cursor),
+                        ...(state.changes.lastSuccessAt !== undefined
+                          ? { lastSuccessAt: state.changes.lastSuccessAt }
+                          : {}),
+                      },
+                    }
+                  : {}),
+                ...(state?.lastError ? { lastError: state.lastError } : {}),
+              }),
+            ),
+          ),
+        { concurrency: "unbounded" },
+      );
+
+      return HttpServerResponse.jsonUnsafe({ connector: connector.name, resources });
+    }).pipe(
+      Effect.catchCause((cause) =>
+        Effect.logWarning(`Status route failed: ${Cause.pretty(cause)}`).pipe(
+          Effect.andThen(
+            Effect.succeed(
+              HttpServerResponse.jsonUnsafe(
+                {
+                  ok: false,
+                  error: "Failed to compute status",
+                } satisfies StatusUnavailableResponse,
+                { status: 500 },
+              ),
+            ),
+          ),
+        ),
+      ),
+    ),
+  );

package/src/status/types.ts

@@ -0,0 +1,33 @@
+import { Schema } from "effect";
+
+import {
+  ResourceErrorSchema as StateResourceErrorSchema,
+  ResourceStateSchema,
+} from "../state-store/schema";
+
+export const SyncStateSchema = Schema.Literals(["pending", "backfilling", "live", "error"]);
+export type SyncState = Schema.Schema.Type<typeof SyncStateSchema>;
+
+/** Safe source failure returned by the status endpoint. */
+export const ResourceErrorSchema = StateResourceErrorSchema;
+export type ResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;
+
+export const ResourceStatusSchema = Schema.Struct({
+  name: Schema.String,
+  state: SyncStateSchema,
+  ...ResourceStateSchema.fields,
+});
+export type ResourceStatus = Schema.Schema.Type<typeof ResourceStatusSchema>;
+
+export const StatusResponseSchema = Schema.Struct({
+  connector: Schema.String,
+  resources: Schema.Array(ResourceStatusSchema),
+});
+export type StatusResponse = Schema.Schema.Type<typeof StatusResponseSchema>;
+
+/** Returned when durable resource status cannot be read. */
+export const StatusUnavailableResponseSchema = Schema.Struct({
+  ok: Schema.Literal(false),
+  error: Schema.String,
+});
+export type StatusUnavailableResponse = Schema.Schema.Type<typeof StatusUnavailableResponseSchema>;

package/src/telemetry.ts

@@ -0,0 +1,288 @@
+import { Cause, Config, Effect, Layer, Metric, Option, type Duration } from "effect";
+import { Headers } from "effect/unstable/http";
+import * as Observability from "effect/unstable/observability";
+
+import { PlatformRuntimeKey } from "./runtime-config/constants";
+
+/** Canonical span names emitted by connector-kit runtime instrumentation. */
+export const SpanName = {
+  batchProcess: "connector.batch.process",
+  webhookDecode: "connector.webhook.decode",
+  webhookHandle: "connector.webhook.handle",
+  publish: "connector.publish",
+  stateSet: "connector.state.set",
+  apiFetch: "connector.api.fetch",
+} as const;
+
+/** Canonical span attribute names. Keep these stable for collector queries. */
+export const Attr = {
+  connectorName: "airfoil.connector.name",
+  connectorResourcesCount: "airfoil.connector.resources.count",
+  resourceName: "airfoil.resource.name",
+  resourceSource: "airfoil.resource.source",
+  batchMutations: "airfoil.batch.mutations",
+  webhookPath: "airfoil.webhook.path",
+  publisherSuccess: "airfoil.publisher.success",
+  stateKey: "airfoil.state.key",
+  apiPath: "airfoil.api.path",
+  errorPhase: "airfoil.error.phase",
+  errorType: "airfoil.error.type",
+  errorMessage: "airfoil.error.message",
+  errorDetails: "airfoil.error.details",
+} as const;
+
+/** Span event names for high-cardinality reproduction details. */
+export const EventName = {
+  batchCheckpoint: "airfoil.batch.checkpoint",
+} as const;
+
+/** Event-only attributes. These are intentionally not span attributes. */
+export const EventAttr = {
+  batchCursor: "airfoil.batch.cursor",
+} as const;
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null;
+
+const maxErrorMessageLength = 500;
+
+const OtlpEnvConfig = Config.all({
+  enabled: Config.boolean(PlatformRuntimeKey.otelEnabled).pipe(Config.withDefault(false)),
+  baseUrl: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpEndpoint)),
+  rawHeaders: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpHeaders)),
+});
+
+const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
+  "authorization",
+  "cookie",
+  "set-cookie",
+  "x-api-key",
+  /api[-_]?key/i,
+  /secret/i,
+  /signature/i,
+  /token/i,
+];
+
+const parseOtelHeaders = (value: string): Record<string, string> =>
+  Object.fromEntries(
+    value
+      .split(",")
+      .map((entry) => entry.trim())
+      .filter(Boolean)
+      .flatMap((entry) => {
+        const separator = entry.indexOf("=");
+        if (separator < 1) return [];
+        return [[entry.slice(0, separator).trim(), entry.slice(separator + 1).trim()]];
+      }),
+  );
+
+const appendPath = (baseUrl: string, path: string) =>
+  `${baseUrl.replace(/\/$/, "")}/${path.replace(/^\//, "")}`;
+
+/**
+ * Best-effort stable error classifier for span metadata.
+ *
+ * Effect/Data tagged errors expose `_tag`; that is the most useful value for
+ * connector bugs (`ConnectorError`, `InvalidWebhookPayloadError`, etc.). For
+ * non-tagged errors we fall back to the JS `Error.name`, then constructor name,
+ * then the primitive type.
+ */
+const errorType = (error: unknown) => {
+  if (isRecord(error) && typeof error._tag === "string") {
+    return String(error._tag);
+  }
+  if (error instanceof Error) {
+    return error.name;
+  }
+  if (isRecord(error) && typeof error.constructor?.name === "string") {
+    return error.constructor.name;
+  }
+  return typeof error;
+};
+
+/**
+ * Human-readable error summary for span metadata.
+ *
+ * Tagged errors in this package carry a `message` field, so this preserves the
+ * important diagnostic text. Unknown object failures are intentionally reduced
+ * to their type; full structured error payloads belong in logs, not span attrs.
+ */
+const errorMessage = (error: unknown) => {
+  const message = (() => {
+    if (isRecord(error) && typeof error.message === "string") {
+      return error.message;
+    }
+    if (typeof error === "string") {
+      return error;
+    }
+    return errorType(error);
+  })();
+
+  if (message.length > maxErrorMessageLength) {
+    return `${message.slice(0, maxErrorMessageLength)}...`;
+  }
+
+  return message;
+};
+
+const errorDetails = (error: unknown): string | undefined => {
+  if (!isRecord(error) || error.cause === undefined) return undefined;
+  return Cause.pretty(Cause.die(error.cause));
+};
+
+/** Annotates the active span with phase-aware error metadata before re-failing. */
+export const annotateError = (phase: string, error: unknown) => {
+  const details = errorDetails(error);
+  return Effect.annotateCurrentSpan({
+    [Attr.errorPhase]: phase,
+    [Attr.errorType]: errorType(error),
+    [Attr.errorMessage]: errorMessage(error),
+    ...(details !== undefined ? { [Attr.errorDetails]: details } : {}),
+  });
+};
+
+/** Adds an event to the active span, or no-ops when no span is active. */
+export const addCurrentSpanEvent = (name: string, attributes?: Record<string, unknown>) =>
+  Effect.currentSpan.pipe(
+    Effect.flatMap((span) =>
+      Effect.clockWith((clock) =>
+        Effect.sync(() => span.event(name, clock.currentTimeNanosUnsafe(), attributes)),
+      ),
+    ),
+    Effect.ignore,
+  );
+
+/** Transport configuration for OTLP trace export. All fields are direct values. */
+export type OtlpTracingConfig = {
+  readonly enabled?: boolean;
+  readonly endpoint?: string;
+  readonly headers?: Record<string, string>;
+};
+
+/** Runtime options — not config-wrappable since RegExp cannot live in Config. */
+export type OtlpTracingOptions = {
+  /** Additional connector-specific header names or regexes to redact from HTTP logs/traces. */
+  readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
+};
+
+export type OtlpMetricsOptions = {
+  readonly exportInterval?: Duration.Input;
+};
+
+const buildOtlpTracingLayer = (config: OtlpTracingConfig, options: OtlpTracingOptions) =>
+  Effect.gen(function* () {
+    const allRedacted = [...defaultRedactedHeaders, ...(options.redactedHeaders ?? [])];
+    const RedactionLayer = Layer.succeed(Headers.CurrentRedactedNames)(allRedacted);
+
+    if (!config.enabled) {
+      return RedactionLayer;
+    }
+
+    if (!config.endpoint) {
+      return yield* Effect.fail(new Error("enabled=true requires endpoint"));
+    }
+
+    yield* Effect.logInfo("telemetry enabled").pipe(
+      Effect.annotateLogs({
+        endpoint: config.endpoint,
+        headers: config.headers ? Object.keys(config.headers) : [],
+      }),
+    );
+
+    return Layer.mergeAll(
+      RedactionLayer,
+      Observability.OtlpTracer.layer({
+        url: appendPath(config.endpoint, "/v1/traces"),
+        headers: config.headers,
+      }).pipe(Layer.provide(Observability.OtlpSerialization.layerJson)),
+    );
+  });
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Takes direct config values — no `ConfigProvider` required.
+ */
+export const layer = (config: OtlpTracingConfig = {}, options: OtlpTracingOptions = {}) =>
+  Layer.unwrap(buildOtlpTracingLayer(config, options));
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Takes Effect Config-wrapped values — reads from `ConfigProvider`.
+ *
+ * @example
+ * Telemetry.layerConfig({
+ *   enabled: Config.boolean("MY_OTEL_ENABLED"),
+ *   endpoint: Config.string("MY_COLLECTOR_URL"),
+ * })
+ */
+export const layerConfig = (
+  config: Config.Wrap<OtlpTracingConfig>,
+  options: OtlpTracingOptions = {},
+) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const resolved = yield* Config.unwrap(config);
+      return yield* buildOtlpTracingLayer(resolved, options);
+    }),
+  );
+
+/**
+ * Trace-only OTLP export with sensitive HTTP header redaction.
+ * Zero-config shortcut that reads the standard `OTEL_*` env var names.
+ *
+ * Reads `OTEL_ENABLED`, `OTEL_EXPORTER_OTLP_ENDPOINT`, and
+ * `OTEL_EXPORTER_OTLP_HEADERS`. Effect's OTLP resource handling reads
+ * `OTEL_SERVICE_NAME`, `OTEL_SERVICE_VERSION`, and `OTEL_RESOURCE_ATTRIBUTES`.
+ */
+export const layerOtlpTracing = (options: OtlpTracingOptions = {}) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const raw = yield* OtlpEnvConfig;
+      return yield* buildOtlpTracingLayer(
+        {
+          enabled: raw.enabled,
+          endpoint: Option.getOrUndefined(raw.baseUrl),
+          headers: Option.isSome(raw.rawHeaders)
+            ? parseOtelHeaders(raw.rawHeaders.value)
+            : undefined,
+        },
+        options,
+      );
+    }),
+  );
+
+export const layerOtlpMetrics = (options: OtlpMetricsOptions = {}) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const raw = yield* OtlpEnvConfig;
+      if (!raw.enabled) return Layer.empty;
+      const endpoint = Option.getOrUndefined(raw.baseUrl);
+      if (!endpoint)
+        return yield* Effect.fail(
+          new Error("OTEL_ENABLED=true requires OTEL_EXPORTER_OTLP_ENDPOINT"),
+        );
+      return Observability.OtlpMetrics.layer({
+        url: appendPath(endpoint, "/v1/metrics"),
+        headers: Option.isSome(raw.rawHeaders) ? parseOtelHeaders(raw.rawHeaders.value) : undefined,
+        exportInterval: options.exportInterval,
+        temporality: "cumulative",
+      }).pipe(Layer.provide(Observability.OtlpSerialization.layerJson));
+    }),
+  );
+
+export const layerOtlp = (options: OtlpTracingOptions & OtlpMetricsOptions = {}) =>
+  Layer.mergeAll(layerOtlpTracing(options), layerOtlpMetrics(options));
+
+export const layerMetricsConsoleDump = (interval: Duration.Input = "30 seconds") =>
+  Layer.effectDiscard(
+    Effect.gen(function* () {
+      yield* Effect.forkScoped(
+        Effect.forever(
+          Metric.dump.pipe(
+            Effect.flatMap((snapshot) => Effect.logInfo(snapshot)),
+            Effect.andThen(Effect.sleep(interval)),
+          ),
+        ),
+      );
+    }),
+  );

package/src/webhook/index.ts

@@ -0,0 +1,8 @@
+import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from "../core/types";
+
+export { router } from "./server";
+export type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from "./types";
+
+export const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(
+  definition: WebhookRouteInput<Resources, Payload>,
+): WebhookRoute<Resources, Payload> => definition;