@useairfoil/connector-kit 0.5.0 → 0.5.2

package/src/runtime-config/index.ts

@@ -0,0 +1,3 @@
+export { RuntimeConfigError, type RuntimeConfigErrorCode } from "./error";
+export { PlatformRuntimeDefault, PlatformRuntimeKey } from "./constants";
+export { layerHosted } from "./layer";

package/src/runtime-config/layer.ts

@@ -0,0 +1,56 @@
+import { Config, ConfigProvider, Effect, FileSystem, Schema } from "effect";
+
+import { PlatformRuntimeKey } from "./constants";
+import { RuntimeConfigError } from "./error";
+
+const providerFromFile: Effect.Effect<
+  ConfigProvider.ConfigProvider,
+  RuntimeConfigError,
+  FileSystem.FileSystem
+> = Effect.gen(function* () {
+  const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(
+    Effect.mapError(
+      (error) =>
+        new RuntimeConfigError({
+          code: "CONFIG_PATH_MISSING",
+          message: `${PlatformRuntimeKey.configPath} is required in hosted mode`,
+          cause: error.cause,
+        }),
+    ),
+  );
+  const fileSystem = yield* FileSystem.FileSystem;
+  const contents = yield* fileSystem.readFileString(configPath).pipe(
+    Effect.mapError(
+      (cause) =>
+        new RuntimeConfigError({
+          code: "CONFIG_FILE_UNREADABLE",
+          message: "Connector config file could not be read",
+          cause,
+        }),
+    ),
+  );
+  const document = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(
+    contents,
+  ).pipe(
+    Effect.mapError(
+      (error) =>
+        new RuntimeConfigError({
+          code: "CONFIG_JSON_INVALID",
+          message: "Connector config is not valid JSON",
+          cause: error,
+        }),
+    ),
+  );
+  return ConfigProvider.fromUnknown(document, { preserveEmptyStrings: true });
+});
+
+/**
+ * Loads the required JSON file and lets environment values override it per key.
+ * Connector field validation remains in the connector's Effect Config.
+ *
+ * @example
+ * ```ts
+ * program.pipe(Effect.provide(layerHosted()));
+ * ```
+ */
+export const layerHosted = () => ConfigProvider.layerAdd(providerFromFile);

package/src/state-store/index.ts

@@ -1,2 +1,3 @@
-export { layerMemory } from "./memory";
 export { StateStore, type StateStoreService } from "./service";
+export { layerMemory } from "./layer-memory";
+export { layerSql } from "./layer-sql";

package/src/state-store/keys.ts

@@ -0,0 +1,22 @@
+import type { StateSource } from "./schema";
+
+type StateComponent = StateSource | `error:${StateSource}`;
+
+export const resourceComponentKey = (resource: string, component: StateComponent): string =>
+  `resource:${resource}:${component}`;
+
+export const resourceErrorKey = (resource: string, source: StateSource): string =>
+  resourceComponentKey(resource, `error:${source}`);
+
+/**
+ * Prefix for all state owned by one connector instance. The platform-owned ID
+ * stays visible so operators can inspect and clean up shared-table rows directly.
+ *
+ * @example
+ * ```ts
+ * connectorInstanceKeyPrefix("team-a/orders");
+ * // "connector-instance:team-a/orders:"
+ * ```
+ */
+export const connectorInstanceKeyPrefix = (connectorInstanceId: string): string =>
+  `connector-instance:${connectorInstanceId}:`;

package/src/state-store/layer-key-value-store.ts

@@ -0,0 +1,132 @@
+import { DateTime, Effect, Layer, Option, Schema } from "effect";
+import { KeyValueStore } from "effect/unstable/persistence";
+
+import { ConnectorError } from "../errors";
+import { resourceComponentKey, resourceErrorKey } from "./keys";
+import {
+  BackfillStateSchema,
+  ChangesStateSchema,
+  ResourceErrorSchema,
+  type PersistedBackfillState,
+  type PersistedChangesState,
+  type PersistedResourceError,
+  type ResourceErrorContext,
+  type StateOperation,
+  type StateSource,
+} from "./schema";
+import { StateStore } from "./service";
+
+const mapStoreError = (operation: "read" | "write" | "clear", resource: string) =>
+  Effect.mapError(
+    (cause: KeyValueStore.KeyValueStoreError | Schema.SchemaError) =>
+      new ConnectorError({
+        message: `Failed to ${operation} state for resource ${resource}`,
+        cause,
+      }),
+  );
+
+const resourceErrorContext = (
+  source: StateSource,
+  operation: StateOperation,
+): ResourceErrorContext => {
+  const sourceName = source === "backfill" ? "Backfill" : "Changes";
+  switch (operation) {
+    case "fetch":
+      return { source, operation, code: "fetch_failed", message: `${sourceName} fetch failed` };
+    case "publish":
+      return { source, operation, code: "publish_failed", message: `${sourceName} publish failed` };
+    case "checkpoint":
+      return {
+        source,
+        operation,
+        code: "checkpoint_failed",
+        message: `${sourceName} checkpoint failed`,
+      };
+  }
+};
+
+const latestError = (
+  backfill: Option.Option<PersistedResourceError>,
+  changes: Option.Option<PersistedResourceError>,
+): Option.Option<PersistedResourceError> => {
+  if (Option.isNone(backfill)) return changes;
+  if (Option.isNone(changes)) return backfill;
+  return backfill.value.at >= changes.value.at ? backfill : changes;
+};
+
+/**
+ * Implements StateStore over any Effect KeyValueStore. `toSchemaStore` owns
+ * serialization and validates persisted values when they are read and written.
+ * The caller owns instance scoping; `layerSql` supplies the production prefix.
+ */
+export const layerKeyValueStore: Layer.Layer<StateStore, never, KeyValueStore.KeyValueStore> =
+  Layer.effect(StateStore)(
+    Effect.gen(function* () {
+      const keyValueStore = yield* KeyValueStore.KeyValueStore;
+      const backfillStore = KeyValueStore.toSchemaStore(keyValueStore, BackfillStateSchema);
+      const changesStore = KeyValueStore.toSchemaStore(keyValueStore, ChangesStateSchema);
+      const errorStore = KeyValueStore.toSchemaStore(keyValueStore, ResourceErrorSchema);
+
+      const getResourceState = (resource: string) =>
+        // Components use separate keys so a cursor update cannot overwrite the
+        // independently managed error or backfill state.
+        Effect.all(
+          {
+            backfill: backfillStore.get(resourceComponentKey(resource, "backfill")),
+            changes: changesStore.get(resourceComponentKey(resource, "changes")),
+            backfillError: errorStore.get(resourceErrorKey(resource, "backfill")),
+            changesError: errorStore.get(resourceErrorKey(resource, "changes")),
+          },
+          { concurrency: "unbounded" },
+        ).pipe(
+          Effect.map(({ backfill, changes, backfillError, changesError }) => {
+            const lastError = latestError(backfillError, changesError);
+            if (Option.isNone(backfill) && Option.isNone(changes) && Option.isNone(lastError)) {
+              return undefined;
+            }
+            return {
+              ...(Option.isSome(backfill) ? { backfill: backfill.value } : {}),
+              ...(Option.isSome(changes) ? { changes: changes.value } : {}),
+              ...(Option.isSome(lastError) ? { lastError: lastError.value } : {}),
+            };
+          }),
+          mapStoreError("read", resource),
+        );
+
+      const setBackfillState = (resource: string, state: PersistedBackfillState) =>
+        backfillStore
+          .set(resourceComponentKey(resource, "backfill"), state)
+          .pipe(mapStoreError("write", resource));
+
+      const setChangesState = (resource: string, state: PersistedChangesState) =>
+        changesStore
+          .set(resourceComponentKey(resource, "changes"), state)
+          .pipe(mapStoreError("write", resource));
+
+      const setResourceError = (resource: string, source: StateSource, operation: StateOperation) =>
+        DateTime.now.pipe(
+          Effect.map(DateTime.formatIso),
+          Effect.flatMap((at) => {
+            const error = resourceErrorContext(source, operation);
+            return errorStore.set(resourceErrorKey(resource, source), {
+              ...error,
+              at,
+            });
+          }),
+          mapStoreError("write", resource),
+        );
+
+      const clearResourceError = (resource: string, source: StateSource) =>
+        errorStore
+          .remove(resourceErrorKey(resource, source))
+          .pipe(mapStoreError("clear", resource));
+
+      return StateStore.of({
+        getResourceState,
+        setBackfillState,
+        setChangesState,
+        setResourceError,
+        clearResourceError,
+      });
+    }),
+  );

package/src/state-store/layer-memory.ts

@@ -0,0 +1,11 @@
+import { Layer } from "effect";
+import { KeyValueStore } from "effect/unstable/persistence";
+
+import type { StateStore } from "./service";
+
+import { layerKeyValueStore } from "./layer-key-value-store";
+
+/** Process-local state for tests and explicit local sandbox commands. */
+export const layerMemory: Layer.Layer<StateStore> = layerKeyValueStore.pipe(
+  Layer.provide(KeyValueStore.layerMemory),
+);

package/src/state-store/layer-sql.ts

@@ -0,0 +1,50 @@
+import { Config, Effect, Layer } from "effect";
+import { KeyValueStore } from "effect/unstable/persistence";
+import { SqlClient } from "effect/unstable/sql";
+
+import type { StateStore } from "./service";
+
+import { PlatformRuntimeDefault, PlatformRuntimeKey } from "../runtime-config/constants";
+import { connectorInstanceKeyPrefix } from "./keys";
+import { layerKeyValueStore } from "./layer-key-value-store";
+
+const scopedKeyValueStore = (connectorInstanceId: string) =>
+  Layer.effect(KeyValueStore.KeyValueStore)(
+    KeyValueStore.KeyValueStore.pipe(
+      Effect.map(KeyValueStore.prefix(connectorInstanceKeyPrefix(connectorInstanceId))),
+    ),
+  );
+
+/**
+ * Shared SQL state using the immutable connector instance ID supplied by
+ * platform-owned Effect configuration. The table defaults to
+ * `_airfoil_connectors_state` and can be overridden with `AIRFOIL_STATE_TABLE`.
+ * Effect's SQL KeyValueStore owns the table, while `KeyValueStore.prefix`
+ * provides application-level isolation between connector instances in that
+ * table. The resulting layer exposes only StateStore, never the unscoped
+ * KeyValueStore.
+ *
+ * @example
+ * ```ts
+ * const StateStoreLive = layerSql().pipe(Layer.provide(PostgresLive));
+ * ```
+ */
+export const layerSql = (): Layer.Layer<StateStore, Config.ConfigError, SqlClient.SqlClient> =>
+  Layer.unwrap(
+    Config.all({
+      connectorInstanceId: Config.nonEmptyString(PlatformRuntimeKey.connectorInstanceId),
+      table: Config.nonEmptyString(PlatformRuntimeKey.stateTable).pipe(
+        Config.withDefault(PlatformRuntimeDefault.stateTable),
+      ),
+    }).pipe(
+      Effect.map(({ connectorInstanceId, table }) =>
+        layerKeyValueStore.pipe(
+          Layer.provide(
+            scopedKeyValueStore(connectorInstanceId).pipe(
+              Layer.provide(KeyValueStore.layerSql({ table })),
+            ),
+          ),
+        ),
+      ),
+    ),
+  );

package/src/state-store/schema.ts

@@ -0,0 +1,57 @@
+import { Schema } from "effect";
+
+const CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);
+
+/** Durable progress for a finite historical backfill. */
+export const BackfillStateSchema = Schema.Struct({
+  cutoff: CursorValueSchema,
+  pageCursor: Schema.optional(CursorValueSchema),
+  completed: Schema.Boolean,
+  lastSuccessAt: Schema.optional(Schema.String),
+});
+
+/** Durable cursor for an ongoing incremental changes stream. */
+export const ChangesStateSchema = Schema.Struct({
+  cursor: CursorValueSchema,
+  lastSuccessAt: Schema.optional(Schema.String),
+});
+
+export const StateSourceSchema = Schema.Literals(["backfill", "changes"]);
+
+export const StateOperationSchema = Schema.Literals(["fetch", "publish", "checkpoint"]);
+
+export const StateErrorCodeSchema = Schema.Literals([
+  "fetch_failed",
+  "publish_failed",
+  "checkpoint_failed",
+]);
+
+/** SDK-owned error context safe to persist and expose through status. */
+export const ResourceErrorContextSchema = Schema.Struct({
+  source: StateSourceSchema,
+  operation: StateOperationSchema,
+  code: StateErrorCodeSchema,
+  message: Schema.String,
+});
+
+/** Stable public error summary persisted for status reporting; never a raw cause. */
+export const ResourceErrorSchema = Schema.Struct({
+  ...ResourceErrorContextSchema.fields,
+  at: Schema.String,
+});
+
+/** Complete persisted state assembled from independently stored components. */
+export const ResourceStateSchema = Schema.Struct({
+  backfill: Schema.optional(BackfillStateSchema),
+  changes: Schema.optional(ChangesStateSchema),
+  lastError: Schema.optional(ResourceErrorSchema),
+});
+
+export type PersistedBackfillState = Schema.Schema.Type<typeof BackfillStateSchema>;
+export type PersistedChangesState = Schema.Schema.Type<typeof ChangesStateSchema>;
+export type StateSource = Schema.Schema.Type<typeof StateSourceSchema>;
+export type StateOperation = Schema.Schema.Type<typeof StateOperationSchema>;
+export type StateErrorCode = Schema.Schema.Type<typeof StateErrorCodeSchema>;
+export type ResourceErrorContext = Schema.Schema.Type<typeof ResourceErrorContextSchema>;
+export type PersistedResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;
+export type PersistedResourceState = Schema.Schema.Type<typeof ResourceStateSchema>;

package/src/state-store/service.ts

@@ -1,18 +1,51 @@
-import { Context, type Effect } from "effect";
+import { Context, Effect } from "effect";
 
-import type { ResourceState } from "../core/types";
-import type { ConnectorError } from "../errors";
+import type {
+  PersistedBackfillState,
+  PersistedChangesState,
+  PersistedResourceState,
+  StateOperation,
+  StateSource,
+} from "./schema";
 
+import { ConnectorError } from "../errors";
+
+/** Connector-facing durable state operations, independent of the storage backend. */
 export interface StateStoreService {
   readonly getResourceState: (
     resource: string,
-  ) => Effect.Effect<ResourceState | undefined, ConnectorError>;
-  readonly setResourceState: (
+  ) => Effect.Effect<PersistedResourceState | undefined, ConnectorError>;
+  readonly setBackfillState: (
+    resource: string,
+    state: PersistedBackfillState,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly setChangesState: (
+    resource: string,
+    state: PersistedChangesState,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly setResourceError: (
+    resource: string,
+    source: StateSource,
+    operation: StateOperation,
+  ) => Effect.Effect<void, ConnectorError>;
+  readonly clearResourceError: (
     resource: string,
-    state: ResourceState,
+    source: StateSource,
   ) => Effect.Effect<void, ConnectorError>;
 }
 
+/**
+ * Effect service used by ingestion and status logic for connector checkpoints.
+ * Production provides `layerSql`; sandboxes and tests may provide `layerMemory`.
+ *
+ * @example
+ * ```ts
+ * const program = Effect.gen(function* () {
+ *   const state = yield* StateStore;
+ *   yield* state.setChangesState("orders", { cursor: "next-page" });
+ * });
+ * ```
+ */
 export class StateStore extends Context.Service<StateStore, StateStoreService>()(
   "@useairfoil/connector-kit/StateStore",
 ) {}

package/src/state-store/state.ts

@@ -0,0 +1,17 @@
+import type { Cursor, ResourceDefinition, ResourceState, SyncState } from "../core/types";
+
+/** Normalizes a cursor value to its persisted/wire representation (never a Date). */
+export const normalizeCursor = (value: Cursor.Value): string | number =>
+  value instanceof Date ? value.toISOString() : value;
+
+/** Derives connector-facing sync state from durable state and resource capabilities. */
+export const deriveSyncState = (
+  resource: ResourceDefinition,
+  state: ResourceState | undefined,
+): SyncState => {
+  if (state?.lastError) return "error";
+  if (!resource.backfill && !resource.changes) return "live";
+  if (!state) return "pending";
+  if (resource.backfill && (!state.backfill || !state.backfill.completed)) return "backfilling";
+  return "live";
+};

package/src/status/index.ts

@@ -0,0 +1,2 @@
+export * from "./route";
+export * from "./types";

package/src/status/route.ts

@@ -0,0 +1,77 @@
+import { Cause, Effect } from "effect";
+import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+
+import type { ConnectorDefinition } from "../core/types";
+import type { ResourceStatus, StatusUnavailableResponse } from "./types";
+
+import { StateStore } from "../state-store";
+import { deriveSyncState, normalizeCursor } from "../state-store/state";
+
+export const statusRoute = (
+  connector: ConnectorDefinition,
+  path: HttpRouter.PathInput = "/status",
+) =>
+  HttpRouter.add(
+    "GET",
+    path,
+    Effect.gen(function* () {
+      const store = yield* StateStore;
+
+      const resources = yield* Effect.forEach(
+        connector.resources,
+        (resource) =>
+          store.getResourceState(resource.name).pipe(
+            Effect.map(
+              (state): ResourceStatus => ({
+                name: resource.name,
+                state: deriveSyncState(resource, state),
+                ...(state?.backfill
+                  ? {
+                      backfill: {
+                        completed: state.backfill.completed,
+                        cutoff: normalizeCursor(state.backfill.cutoff),
+                        ...(state.backfill.pageCursor !== undefined
+                          ? { pageCursor: normalizeCursor(state.backfill.pageCursor) }
+                          : {}),
+                        ...(state.backfill.lastSuccessAt !== undefined
+                          ? { lastSuccessAt: state.backfill.lastSuccessAt }
+                          : {}),
+                      },
+                    }
+                  : {}),
+                ...(state?.changes
+                  ? {
+                      changes: {
+                        cursor: normalizeCursor(state.changes.cursor),
+                        ...(state.changes.lastSuccessAt !== undefined
+                          ? { lastSuccessAt: state.changes.lastSuccessAt }
+                          : {}),
+                      },
+                    }
+                  : {}),
+                ...(state?.lastError ? { lastError: state.lastError } : {}),
+              }),
+            ),
+          ),
+        { concurrency: "unbounded" },
+      );
+
+      return HttpServerResponse.jsonUnsafe({ connector: connector.name, resources });
+    }).pipe(
+      Effect.catchCause((cause) =>
+        Effect.logWarning(`Status route failed: ${Cause.pretty(cause)}`).pipe(
+          Effect.andThen(
+            Effect.succeed(
+              HttpServerResponse.jsonUnsafe(
+                {
+                  ok: false,
+                  error: "Failed to compute status",
+                } satisfies StatusUnavailableResponse,
+                { status: 500 },
+              ),
+            ),
+          ),
+        ),
+      ),
+    ),
+  );

package/src/status/types.ts

@@ -0,0 +1,33 @@
+import { Schema } from "effect";
+
+import {
+  ResourceErrorSchema as StateResourceErrorSchema,
+  ResourceStateSchema,
+} from "../state-store/schema";
+
+export const SyncStateSchema = Schema.Literals(["pending", "backfilling", "live", "error"]);
+export type SyncState = Schema.Schema.Type<typeof SyncStateSchema>;
+
+/** Safe source failure returned by the status endpoint. */
+export const ResourceErrorSchema = StateResourceErrorSchema;
+export type ResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;
+
+export const ResourceStatusSchema = Schema.Struct({
+  name: Schema.String,
+  state: SyncStateSchema,
+  ...ResourceStateSchema.fields,
+});
+export type ResourceStatus = Schema.Schema.Type<typeof ResourceStatusSchema>;
+
+export const StatusResponseSchema = Schema.Struct({
+  connector: Schema.String,
+  resources: Schema.Array(ResourceStatusSchema),
+});
+export type StatusResponse = Schema.Schema.Type<typeof StatusResponseSchema>;
+
+/** Returned when durable resource status cannot be read. */
+export const StatusUnavailableResponseSchema = Schema.Struct({
+  ok: Schema.Literal(false),
+  error: Schema.String,
+});
+export type StatusUnavailableResponse = Schema.Schema.Type<typeof StatusUnavailableResponseSchema>;

package/src/telemetry.ts

@@ -1,7 +1,9 @@
-import { Cause, Config, Effect, Layer, Option } from "effect";
+import { Cause, Config, Effect, Layer, Metric, Option, type Duration } from "effect";
 import { Headers } from "effect/unstable/http";
 import * as Observability from "effect/unstable/observability";
 
+import { PlatformRuntimeKey } from "./runtime-config/constants";
+
 /** Canonical span names emitted by connector-kit runtime instrumentation. */
 export const SpanName = {
   batchProcess: "connector.batch.process",
@@ -45,9 +47,9 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
 const maxErrorMessageLength = 500;
 
 const OtlpEnvConfig = Config.all({
-  enabled: Config.boolean("OTEL_ENABLED").pipe(Config.withDefault(false)),
-  baseUrl: Config.option(Config.string("OTEL_EXPORTER_OTLP_ENDPOINT")),
-  rawHeaders: Config.option(Config.string("OTEL_EXPORTER_OTLP_HEADERS")),
+  enabled: Config.boolean(PlatformRuntimeKey.otelEnabled).pipe(Config.withDefault(false)),
+  baseUrl: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpEndpoint)),
+  rawHeaders: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpHeaders)),
 });
 
 const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
@@ -163,6 +165,10 @@ export type OtlpTracingOptions = {
   readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
 };
 
+export type OtlpMetricsOptions = {
+  readonly exportInterval?: Duration.Input;
+};
+
 const buildOtlpTracingLayer = (config: OtlpTracingConfig, options: OtlpTracingOptions) =>
   Effect.gen(function* () {
     const allRedacted = [...defaultRedactedHeaders, ...(options.redactedHeaders ?? [])];
@@ -244,3 +250,39 @@ export const layerOtlpTracing = (options: OtlpTracingOptions = {}) =>
       );
     }),
   );
+
+export const layerOtlpMetrics = (options: OtlpMetricsOptions = {}) =>
+  Layer.unwrap(
+    Effect.gen(function* () {
+      const raw = yield* OtlpEnvConfig;
+      if (!raw.enabled) return Layer.empty;
+      const endpoint = Option.getOrUndefined(raw.baseUrl);
+      if (!endpoint)
+        return yield* Effect.fail(
+          new Error("OTEL_ENABLED=true requires OTEL_EXPORTER_OTLP_ENDPOINT"),
+        );
+      return Observability.OtlpMetrics.layer({
+        url: appendPath(endpoint, "/v1/metrics"),
+        headers: Option.isSome(raw.rawHeaders) ? parseOtelHeaders(raw.rawHeaders.value) : undefined,
+        exportInterval: options.exportInterval,
+        temporality: "cumulative",
+      }).pipe(Layer.provide(Observability.OtlpSerialization.layerJson));
+    }),
+  );
+
+export const layerOtlp = (options: OtlpTracingOptions & OtlpMetricsOptions = {}) =>
+  Layer.mergeAll(layerOtlpTracing(options), layerOtlpMetrics(options));
+
+export const layerMetricsConsoleDump = (interval: Duration.Input = "30 seconds") =>
+  Layer.effectDiscard(
+    Effect.gen(function* () {
+      yield* Effect.forkScoped(
+        Effect.forever(
+          Metric.dump.pipe(
+            Effect.flatMap((snapshot) => Effect.logInfo(snapshot)),
+            Effect.andThen(Effect.sleep(interval)),
+          ),
+        ),
+      );
+    }),
+  );