@useairfoil/connector-kit 0.5.0 → 0.5.1

package/METRICS.md

@@ -0,0 +1,78 @@
+# Connector Metrics And Status
+
+Connector Kit emits standard metrics through Effect metrics. Production runtimes should use `Telemetry.layerOtlp()` with `OTEL_ENABLED=true`, `OTEL_EXPORTER_OTLP_ENDPOINT`, and optional `OTEL_EXPORTER_OTLP_HEADERS`. HTTP runtimes expose shallow process health at `GET /health`, Prometheus text at `GET /metrics`, and durable sync state at `GET /status`. Sandboxes can also merge `Telemetry.layerMetricsConsoleDump()` for periodic log output.
+
+## Metrics
+
+| Name                                               | Type            | Unit            | Attributes                                   |
+| -------------------------------------------------- | --------------- | --------------- | -------------------------------------------- |
+| `airfoil_connector_entities_upserted_total`        | counter         | mutations       | `connector`, `resource`, `source`            |
+| `airfoil_connector_entities_deleted_total`         | counter         | mutations       | `connector`, `resource`, `source`            |
+| `airfoil_connector_batches_total`                  | counter         | batches         | `connector`, `resource`, `source`, `outcome` |
+| `airfoil_connector_batch_size`                     | histogram       | mutations/batch | `connector`, `resource`, `source`            |
+| `airfoil_connector_publish_duration_ms`            | histogram timer | ms              | `connector`, `resource`, `source`            |
+| `airfoil_connector_webhook_requests_total`         | counter         | requests        | `connector`, `path`, `outcome`               |
+| `airfoil_connector_webhook_queue_depth`            | gauge           | batches         | `connector`                                  |
+| `airfoil_connector_sync_state`                     | gauge           | none            | `connector`, `resource`, `state`             |
+| `airfoil_connector_last_success_timestamp_seconds` | gauge           | Unix seconds    | `connector`, `resource`, `source`            |
+| `airfoil_connector_api_retries_total`              | counter         | retries         | `connector`, `reason`                        |
+
+Batch `source` is `backfill`, `changes`, or `webhook`. Batch `outcome` is `accepted`, `rejected`, or `error`. Webhook `outcome` is `ok`, `read_error`, `invalid_json`, `invalid_payload`, `rejected`, or `handler_error`. Sync `state` is `pending`, `backfilling`, `live`, or `error`.
+
+API retry `reason` is `transport`, `timeout`, `rate_limit`, or `server_error`. The counter increments only when another request will be attempted. Provider-side limiter waits without a failed request are not retries.
+
+Counters are activity since process start, not entity population. Use `increase()` or `rate()` and let Prometheus/OTLP consumers handle process restarts. `sum(upserts) - sum(deletes)` is connector activity, not the current table size; current totals belong in Wings/data-lake tables.
+
+The last-success gauge is restored from durable state after restart and is `0` until the source commits its first checkpoint. Calculate source freshness in Prometheus:
+
+```promql
+time() - airfoil_connector_last_success_timestamp_seconds
+```
+
+## Sync Status
+
+HTTP connector runtimes expose `GET /status` by default. The response shape is decoded by `Status.StatusResponseSchema`:
+
+```json
+{
+  "connector": "producer-example",
+  "resources": [
+    {
+      "name": "products",
+      "state": "live",
+      "backfill": {
+        "completed": true,
+        "cutoff": "2026-01-01T00:00:00Z",
+        "lastSuccessAt": "2026-01-01T00:05:00Z"
+      },
+      "changes": {
+        "cursor": "2026-01-01T00:10:00Z",
+        "lastSuccessAt": "2026-01-01T00:10:01Z"
+      }
+    }
+  ]
+}
+```
+
+`ResourceStatusSchema`, `ResourceErrorSchema`, and `StatusUnavailableResponseSchema` are also exported from `@useairfoil/connector-kit/status` with their derived TypeScript types.
+
+A resource with no persisted state is `pending`; an incomplete backfill is `backfilling`; completed backfill or changes-only resources are `live`. `lastSuccessAt` advances in the same durable write as its checkpoint and only after Wings accepts the batch.
+
+Backfill or changes failures set `error` with a structured `lastError`. The error includes `source`, `operation`, a stable `code` and SDK-owned `message`, plus `at`; it never contains the raw Effect cause. Backfill and changes errors are stored independently, and `/status` returns the newer outstanding error when both exist.
+
+When an expected source failure reaches the ingestion engine after any dependency retries, that source is parked until the process restarts. Other resources, webhooks, and HTTP routes keep running. Defects and interruption still terminate the runtime. `GET /health` only proves that the HTTP process is serving requests; provider, Wings, or PostgreSQL failures are reported through status and metrics instead.
+
+## Wings Contract
+
+Import pure connector manifests from first-party producer packages, for example `@useairfoil/producer-polar/manifest`, through a registry. Use `Manifest.decodeConfig(manifest, input)` for canonical server-side validation and `Schema.toStandardSchemaV1(Manifest.configSchema(manifest))` for form resolvers. The schema accepts JSON-shaped config and browser form-shaped values, including stringified numbers/booleans and empty optional fields. Treat `field.required` and `field.default` independently: the canonical decoder treats an empty optional field as omitted and resolves its default, while an optional field without a default remains absent. Store user-facing connector config by manifest field `name`, then use `Manifest.toRuntimeDocument(...)` to map it to `field.runtimeKey` names. Hosted pods mount that complete JSON document read-only and point `AIRFOIL_CONFIG_PATH` at it; they do not receive one environment variable per user field. Secret rendering/redaction comes from `field.secret`, and existing secret values must not be returned to the browser. Platform-owned runtime config such as Wings host, namespace, tables, PostgreSQL bindings, webhook ports, and telemetry remains infrastructure-owned environment/Secret configuration outside the manifest document.
+
+```ts
+import * as Manifest from "@useairfoil/connector-kit/manifest";
+import { manifest as polarManifest } from "@useairfoil/producer-polar/manifest";
+import { Schema } from "effect";
+
+const schema = Schema.toStandardSchemaV1(Manifest.configSchema(polarManifest));
+const result = await schema["~standard"].validate(formValues);
+```
+
+Poll `GET /status` and decode with `Status.StatusResponseSchema`. Chart ingestion with `increase(airfoil_connector_entities_upserted_total[...])`, alert on `airfoil_connector_sync_state{state="error"} == 1`, and alert on non-`ok` webhook outcomes.

package/README.md

@@ -12,17 +12,19 @@ Toolkit for building Airfoil producer connectors with Effect.
 - `Ingestion.run(...)` runs resource ingestion and checkpoints after accepted publishes.
 - `Publisher.layerConsole` logs accepted batches locally.
 - `Publisher.layerWings(...)` publishes resource mutations into Wings tables.
+- `ConnectorApp.check(...)` validates selected resources before provisioning.
 - `ConnectorApp.start(...)` starts a complete Node HTTP connector process.
+- `Auth.makeTokenCache(...)` caches short-lived provider access tokens in memory.
 
 The package is intentionally Layer-oriented: connector code defines resources and routes, while entrypoints provide runtime dependencies such as publishers, state stores, telemetry, and API clients.
 
 ## Install
 
 ```bash
-pnpm add @useairfoil/connector-kit effect@beta
+pnpm add @useairfoil/connector-kit effect@^4.0.0-rc.108
 ```
 
-This repo currently uses Effect v4 beta. In workspace packages, prefer the workspace catalog (`"effect": "catalog:"`) so all packages stay on the pinned beta version.
+This repo currently uses the Effect v4 release candidate. In workspace packages, prefer the workspace catalog (`"effect": "catalog:"`) so all packages stay on the supported RC version.
 
 ## Package Shape
 
@@ -30,15 +32,20 @@ The package root exposes the common connector authoring API:
 
 ```ts
 import {
+  Auth,
   Connector,
   ConnectorApp,
   ConnectorError,
   Cursor,
   Fetch,
   Ingestion,
+  Manifest,
+  Metrics,
   Publisher,
   Resource,
+  RuntimeConfig,
   StateStore,
+  Status,
   Telemetry,
   Webhook,
 } from "@useairfoil/connector-kit";
@@ -46,15 +53,139 @@ import {
 
 Subpath exports are available for runtime domains:
 
+- `@useairfoil/connector-kit/auth`
 - `@useairfoil/connector-kit/ingestion`
 - `@useairfoil/connector-kit/connector-app`
+- `@useairfoil/connector-kit/manifest`
+- `@useairfoil/connector-kit/metrics`
 - `@useairfoil/connector-kit/publisher`
+- `@useairfoil/connector-kit/runtime-config`
 - `@useairfoil/connector-kit/state-store`
+- `@useairfoil/connector-kit/status`
 - `@useairfoil/connector-kit/webhook`
 - `@useairfoil/connector-kit/errors`
 
 Core definition helpers and `Telemetry` are intentionally root-only.
 
+## Provider Credentials
+
+`Auth.makeTokenCache(...)` wraps a provider token exchange with Effect's cache. Acquisition stays lazy, concurrent callers share one exchange, and failures are not cached. The default refresh point is halfway through the provider lifetime.
+
+```ts
+import { Auth } from "@useairfoil/connector-kit";
+import { Duration, Effect, Redacted } from "effect";
+
+const tokenCache =
+  yield *
+  Auth.makeTokenCache({
+    acquire: exchangeCredentials.pipe(
+      Effect.map(({ accessToken, expiresInSeconds }) => ({
+        value: Redacted.make(accessToken),
+        expiresIn: Duration.seconds(expiresInSeconds),
+      })),
+    ),
+  });
+
+const accessToken = yield * tokenCache.get;
+```
+
+The cache is process-local and does not use `StateStore`. It does not add retries or background refresh. The first request after the refresh point performs the next exchange.
+
+## Manifests
+
+Connector manifests are pure metadata for UI, Wings deployment, and validation. Put user-facing connector config in `src/manifest.ts` and keep platform-owned runtime config, such as ports, Wings host, table names, and OTEL flags, as normal Effect `Config` in entrypoints.
+
+```ts
+import * as Manifest from "@useairfoil/connector-kit/manifest";
+
+export const ExampleConfigDef = Manifest.defineConfig({
+  apiBaseUrl: Manifest.string({
+    runtimeKey: "EXAMPLE_API_BASE_URL",
+    required: false,
+    default: "https://api.example.test",
+  }),
+  apiToken: Manifest.secret({ runtimeKey: "EXAMPLE_API_TOKEN" }),
+  maxRetries: Manifest.number({
+    runtimeKey: "EXAMPLE_MAX_RETRIES",
+    default: 5,
+    integer: true,
+    minimum: 0,
+  }),
+  webhookSecret: Manifest.optional(Manifest.secret({ runtimeKey: "EXAMPLE_WEBHOOK_SECRET" })),
+});
+
+export type ExampleConfig = Manifest.ConfigValuesOf<typeof ExampleConfigDef>;
+
+export const manifest = Manifest.define({
+  name: "producer-example",
+  title: "Example",
+  config: ExampleConfigDef.spec,
+  resources: [{ name: "posts", capabilities: ["backfill", "webhook"] }],
+});
+```
+
+`ExampleConfigDef.config` is the runtime Effect config. `ExampleConfigDef.spec` is serializable manifest metadata. `required` defaults to `true`. Number fields may set `integer` and `minimum`; Effect Schema applies the same constraints to runtime and form values. Use `required: false` with a default when the form may omit a field but the runtime must always receive a concrete value. Use `Manifest.optional(...)` when the runtime result should be an `Option`. Secret fields cannot declare defaults, so optional secrets use `Manifest.optional(Manifest.secret(...))`.
+
+Use `Manifest.configSchema(manifest)` for form validation and `Schema.toStandardSchemaV1(...)` for standard-schema form resolvers. The schema accepts normal JSON values and browser form-shaped values: number fields may arrive as strings, boolean fields may arrive as booleans or `"true"`/`"false"`, optional fields may be omitted or submitted as `""`, and required strings reject `""`.
+
+Use `Manifest.decodeConfig(manifest, input)` as the canonical server-side decoder. It rejects undeclared fields, treats `""` on a declared optional field as not supplied, and then applies manifest defaults. Therefore clearing or omitting a defaulted optional control means “use the manifest default”; it does not disable that setting. Model disabling as an explicit select/boolean/nullable contract instead of overloading an empty string. An optional field without a default remains absent.
+
+Frontend/backend validation example:
+
+```ts
+import * as Manifest from "@useairfoil/connector-kit/manifest";
+import { manifest as polarManifest } from "@useairfoil/producer-polar/manifest";
+import { Schema } from "effect";
+
+const configSchema = Manifest.configSchema(polarManifest);
+const formSchema = Schema.toStandardSchemaV1(configSchema);
+
+const result = await formSchema["~standard"].validate({
+  accessToken: "polar_oat_xxx",
+  apiBaseUrl: "https://api.polar.sh/v1/",
+  organizationId: "",
+});
+
+if ("issues" in result) {
+  // Show validation issues in your form.
+} else {
+  // Store by manifest field name: accessToken, apiBaseUrl, organizationId.
+  // Secrets should be persisted server-side and not echoed back to the browser.
+  const values = result.value;
+}
+```
+
+Render fields from `manifest.config`: use `field.name` as the form/storage key, `field.description` for help text, `field.secret` for password inputs/redaction, `field.values` for selects, `field.default` for the documented fallback, `field.required` for required markers, and number `integer` and `minimum` values for input constraints. The UI may show a default as an initial value or clearly labelled placeholder, but clearing it still resolves to that default during canonical decoding. Decode submitted values with `Manifest.decodeConfig(...)`, then map the canonical logical object to its flat runtime document with `Manifest.toRuntimeDocument(...)`. Each `field.runtimeKey` is an Effect Config key, not an instruction to create one Pod environment variable.
+
+Hosted connectors receive the runtime document as a read-only JSON file. `AIRFOIL_CONFIG_PATH` selects the file, and `RuntimeConfig.layerHosted()` adds the file beneath Effect's existing environment provider. The file is required and must contain valid JSON; each connector's Effect Config validates the fields it reads. Local sandboxes use Effect's default environment provider. Platform-owned config such as Wings host, namespace, table names, PostgreSQL settings, ports, and OTEL settings stays outside the manifest and outside the connector JSON.
+
+### Pre-provision Checks
+
+The dashboard backend imports each connector package and runs `ConnectorApp.check(...)` after manifest decoding but before persisting secrets or creating database and Kubernetes resources. Validate submitted resource names against `manifest.resources` at the request boundary before calling the typed check API. Replace the backend process provider with the submitted runtime document so unrelated backend environment variables cannot satisfy connector configuration:
+
+```ts
+import { ConnectorApp, Manifest } from "@useairfoil/connector-kit";
+import { PolarConnector, manifest } from "@useairfoil/producer-polar";
+import { ConfigProvider, Effect } from "effect";
+import { FetchHttpClient } from "effect/unstable/http";
+
+const values = yield * Manifest.decodeConfig(manifest, submittedConfig);
+const runtimeDocument = Manifest.toRuntimeDocument(manifest, values);
+const provider = ConfigProvider.fromUnknown(runtimeDocument, { preserveEmptyStrings: true });
+
+const result =
+  yield *
+  ConnectorApp.check(
+    PolarConnector.PolarConnector,
+    PolarConnector.layerConfig(PolarConnector.PolarConfigDef.config),
+    { resources: ["customers", "orders"] },
+  ).pipe(Effect.provide(ConfigProvider.layer(provider)), Effect.provide(FetchHttpClient.layer));
+```
+
+Every resource declares a required, read-only `check` effect. Layer/configuration failures are returned for every selected resource; resource-specific failures are returned only under that resource name. Expected failures are values (`{ _tag: "error", message }`), while defects and interruption remain Effect failures. Provision only when all selected results are `{ _tag: "ok" }`. Checks must never publish, checkpoint, or mutate the upstream provider.
+
+`RuntimeConfig.PlatformRuntimeKey` is the canonical shared-key contract used by hosted config, StateStore, telemetry, Wings, and PostgreSQL bootstrap code. `Manifest.defineConfig` rejects only those exact shared values; it does not impose connector-specific naming, uniqueness, or metadata policy. Connector packages are first-party code, so their table, port, field, and resource naming remains the connector author's responsibility.
+
 ## Resource Model
 
 Resources emit mutations, not raw rows. Connector Kit currently supports `Resource.entity(...)` for Wings table-backed resources.
@@ -65,6 +196,7 @@ const Posts = Resource.entity({
   schema: PostSchema,
   key: "id",
   version: "updatedAt",
+  check: api.fetchPosts({ page: 1, limit: 1 }).pipe(Effect.asVoid),
   backfill: Fetch.page({
     pageCursor: Cursor.number(),
     cutoff: Cursor.isoDateTime(),
@@ -118,6 +250,7 @@ const Posts = Resource.entity({
   schema: PostSchema,
   key: "id",
   version: "updatedAt",
+  check: Effect.void,
   webhook: Resource.webhook({
     schema: PostEventSchema,
     handler: ({ payload }) => Effect.succeed([Resource.upsert(payload.data)]),
@@ -173,9 +306,36 @@ Current runtime behavior:
 - checkpoints only after an accepted publish ACK
 - does not checkpoint rejected publishes
 - allows empty accepted batches to advance state
+- stores `lastSuccessAt` with each durable checkpoint
 - persists resource state through `StateStore.StateStore`
+- parks a source when an expected failure reaches the ingestion engine, after any dependency retries
+- keeps other sources, webhooks, and HTTP routes running when one source is parked
+
+Parked sources resume after process restart or configuration redeployment. Defects and interruption are not isolated and still stop the runtime.
+
+`Ingestion.run` depends only on the typed Airfoil `StateStore`; it does not expose the underlying Effect `KeyValueStore`. Provide `StateStore.layerMemory` for development/tests. Production entrypoints provide `StateStore.layerSql()`, which privately composes Effect's `KeyValueStore.layerSql` and scopes every key with the non-empty `AIRFOIL_CONNECTOR_INSTANCE_ID`. The platform ID is opaque and is not required to be a UUID.
+
+State keys use plain colon-delimited segments and are intentionally inspectable; they are not escaped or encoded. The trusted backend must issue connector instance IDs without `:`, and connector authors must keep resource names free of `:`. Connector Kit currently relies on that naming contract rather than adding runtime validation.
 
-Use `StateStore.layerMemory` for development and tests, or provide your own durable `StateStore` implementation.
+The shared SQL table defaults to `_airfoil_connectors_state`. Set `AIRFOIL_STATE_TABLE` only to override that default. `POSTGRES_CONNECTION_STRING` is consumed by the PostgreSQL client layer, not by `StateStore` itself:
+
+```ts
+import { PgClient } from "@effect/sql-pg";
+import { RuntimeConfig, StateStore } from "@useairfoil/connector-kit";
+import { Config, Layer, Schema } from "effect";
+
+const PostgresLive = PgClient.layerConfig({
+  url: Config.schema(
+    Schema.Redacted(Schema.NonEmptyString),
+    RuntimeConfig.PlatformRuntimeKey.postgresConnectionString,
+  ),
+  maxConnections: Config.succeed(2),
+});
+
+const StateStoreLive = StateStore.layerSql().pipe(Layer.provide(PostgresLive));
+```
+
+The platform injects the instance ID and Secret-backed PostgreSQL connection string used by `PgClient`, plus an optional table-name override. Connector business code receives `StateStore` but does not receive the connector instance ID, raw SQL client, or global clear/size operations.
 
 ## Publisher
 
@@ -224,7 +384,7 @@ import {
   StateStore,
   Telemetry,
 } from "@useairfoil/connector-kit";
-import { ConfigProvider, Effect, Layer, Schema } from "effect";
+import { Effect, Layer, Schema } from "effect";
 import { FetchHttpClient } from "effect/unstable/http";
 
 const PostSchema = Schema.Struct({
@@ -240,6 +400,7 @@ const Posts = Resource.entity({
   schema: PostSchema,
   key: "id",
   version: "updatedAt",
+  check: Effect.void,
   backfill: Fetch.page({
     pageCursor: Cursor.number(),
     cutoff: Cursor.isoDateTime(),
@@ -259,29 +420,36 @@ const connector = Connector.define({
   resources: [Posts],
 });
 
-const EnvLayer = Layer.mergeAll(
-  FetchHttpClient.layer,
-  Layer.succeed(ConfigProvider.ConfigProvider, ConfigProvider.fromEnv()),
-);
+const BootstrapLayer = FetchHttpClient.layer;
 
-const TelemetryLayer = Telemetry.layerOtlpTracing().pipe(Layer.provide(EnvLayer));
-const runtimeLayer = Layer.mergeAll(StateStore.layerMemory, Publisher.layerConsole, TelemetryLayer);
+const TelemetryLayer = Telemetry.layerOtlp().pipe(Layer.provide(BootstrapLayer));
+const RuntimeLayer = Layer.mergeAll(
+  BootstrapLayer,
+  StateStore.layerMemory,
+  Publisher.layerConsole,
+  TelemetryLayer,
+);
 
 Effect.scoped(ConnectorApp.start(connector, { port: 8080 })).pipe(
-  Effect.provide(runtimeLayer),
+  Effect.provide(RuntimeLayer),
   NodeRuntime.runMain,
 );
 ```
 
 ## Telemetry
 
-`Telemetry` contains connector-kit span names, span attributes, error annotation helpers, and OTLP tracing layers.
+`Telemetry` contains connector-kit span names, span attributes, error annotation helpers, OTLP tracing layers, and OTLP metrics layers. HTTP connector runtimes expose shallow process health at `GET /health`, Prometheus metrics at `GET /metrics`, and durable sync state at `GET /status` by default. Provider, Wings, and PostgreSQL failures do not make `/health` fail.
+
+`/status` includes `lastSuccessAt` for each checkpointed backfill or changes source. Use `time() - airfoil_connector_last_success_timestamp_seconds` to monitor source freshness without interpreting provider cursors.
 
 Common entry points:
 
 - `Telemetry.layer(config, options?)`
 - `Telemetry.layerConfig(config, options?)`
-- `Telemetry.layerOtlpTracing(options?)`
+- `Telemetry.layerOtlpTracing(options?)` for OTLP traces
+- `Telemetry.layerOtlpMetrics(options?)` for OTLP metrics
+- `Telemetry.layerOtlp(options?)` for both traces and metrics
+- `Telemetry.layerMetricsConsoleDump(interval?)`
 
 Default redacted headers:
 
@@ -296,14 +464,14 @@ Default redacted headers:
 
 Telemetry environment variables:
 
-| Variable                      | Read by       | Description                                                                                                                |
-| ----------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------- |
-| `OTEL_ENABLED`                | Connector Kit | Enables trace export when `true`. Defaults to `false`.                                                                     |
-| `OTEL_EXPORTER_OTLP_ENDPOINT` | Connector Kit | OTLP base URL, for example `http://localhost:4318`. Connector Kit appends `/v1/traces`. Required when `OTEL_ENABLED=true`. |
-| `OTEL_EXPORTER_OTLP_HEADERS`  | Connector Kit | Optional comma-separated headers, for example `Authorization=Bearer <token>,X-Axiom-Dataset=<dataset>`.                    |
-| `OTEL_SERVICE_NAME`           | Effect        | Service name resource attribute, for example `producer-shopify`.                                                           |
-| `OTEL_SERVICE_VERSION`        | Effect        | Optional service version resource attribute.                                                                               |
-| `OTEL_RESOURCE_ATTRIBUTES`    | Effect        | Optional comma-separated resource attributes, for example `deployment.environment=production,team=data`.                   |
+| Variable                      | Read by       | Description                                                                                                                                            |
+| ----------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `OTEL_ENABLED`                | Connector Kit | Enables OTLP export when `true`. Defaults to `false`.                                                                                                  |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | Connector Kit | OTLP base URL, for example `http://localhost:4318`. Connector Kit appends `/v1/traces` and `/v1/metrics` as needed. Required when `OTEL_ENABLED=true`. |
+| `OTEL_EXPORTER_OTLP_HEADERS`  | Connector Kit | Optional comma-separated headers, for example `Authorization=Bearer <token>,X-Axiom-Dataset=<dataset>`.                                                |
+| `OTEL_SERVICE_NAME`           | Effect        | Service name resource attribute, for example `producer-shopify`.                                                                                       |
+| `OTEL_SERVICE_VERSION`        | Effect        | Optional service version resource attribute.                                                                                                           |
+| `OTEL_RESOURCE_ATTRIBUTES`    | Effect        | Optional comma-separated resource attributes, for example `deployment.environment=production,team=data`.                                               |
 
 ## Testing
 
@@ -311,6 +479,7 @@ For tests, the most common setup is:
 
 - `StateStore.layerMemory` for state
 - a small in-memory `Publisher.Publisher` test layer
+- `ConnectorApp.check(...)` with an in-memory `ConfigProvider` for pre-provision validation
 - `Ingestion.run(...)` or `ConnectorApp.start(...)` inside `Effect.scoped`
 
 Keep HTTP recording concerns outside connector logic by providing a VCR-backed `HttpClient` Layer.

package/dist/auth/index.d.ts

@@ -0,0 +1,2 @@
+import { i as makeTokenCache, n as ExpiringToken, r as TokenCache } from "../index-CTHVudvQ.js";
+export { ExpiringToken, TokenCache, makeTokenCache };

package/dist/auth/index.js

@@ -0,0 +1,3 @@
+import { n as makeTokenCache } from "../auth-DvU_CZ7C.js";
+
+export { makeTokenCache };

package/dist/auth-DvU_CZ7C.js

@@ -0,0 +1,42 @@
+import { t as __export } from "./chunk-Bp6m_JJh.js";
+import { Cache, Duration, Effect, Exit, Number, Redacted } from "effect";
+
+//#region src/auth/token-cache.ts
+/**
+* Creates a lazy, in-memory token cache around an acquisition effect.
+*
+* `refreshFactor` controls how much of the provider lifetime is cached. It is
+* clamped between `0` and `1`.
+*
+* @example
+* ```ts
+* const cache = yield* makeTokenCache({
+*   acquire: exchangeCredentials,
+*   refreshFactor: 0.5,
+* })
+*
+* const token = yield* cache.get
+* ```
+*/
+const makeTokenCache = (options) => Effect.gen(function* () {
+	const refreshFactor = Number.clamp(options.refreshFactor ?? .5, {
+		minimum: 0,
+		maximum: 1
+	});
+	const cache = yield* Cache.makeWith((_) => options.acquire, {
+		capacity: 1,
+		timeToLive: (exit) => Exit.isSuccess(exit) ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor) : Duration.zero
+	});
+	return {
+		get: Cache.get(cache, void 0).pipe(Effect.map((token) => token.value)),
+		invalidate: Cache.invalidate(cache, void 0)
+	};
+});
+
+//#endregion
+//#region src/auth/index.ts
+var auth_exports = /* @__PURE__ */ __export({ makeTokenCache: () => makeTokenCache });
+
+//#endregion
+export { makeTokenCache as n, auth_exports as t };
+//# sourceMappingURL=auth-DvU_CZ7C.js.map

package/dist/auth-DvU_CZ7C.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth-DvU_CZ7C.js","names":[],"sources":["../src/auth/token-cache.ts","../src/auth/index.ts"],"sourcesContent":["import { Cache, Duration, Effect, Exit, Number, Redacted } from \"effect\";\n\nexport interface ExpiringToken {\n  /** The credential kept in memory by the cache. */\n  readonly value: Redacted.Redacted<string>;\n  /** The lifetime reported by the provider. */\n  readonly expiresIn: Duration.Duration;\n}\n\nexport interface TokenCache<E> {\n  /** Returns the cached token or performs one shared acquisition. */\n  readonly get: Effect.Effect<Redacted.Redacted<string>, E>;\n  /** Removes the cached token so the next `get` reacquires it. */\n  readonly invalidate: Effect.Effect<void>;\n}\n\n/**\n * Creates a lazy, in-memory token cache around an acquisition effect.\n *\n * `refreshFactor` controls how much of the provider lifetime is cached. It is\n * clamped between `0` and `1`.\n *\n * @example\n * ```ts\n * const cache = yield* makeTokenCache({\n *   acquire: exchangeCredentials,\n *   refreshFactor: 0.5,\n * })\n *\n * const token = yield* cache.get\n * ```\n */\nexport const makeTokenCache = <E, R>(options: {\n  readonly acquire: Effect.Effect<ExpiringToken, E, R>;\n  readonly refreshFactor?: number;\n}): Effect.Effect<TokenCache<E>, never, R> =>\n  Effect.gen(function* () {\n    const refreshFactor = Number.clamp(options.refreshFactor ?? 0.5, {\n      minimum: 0,\n      maximum: 1,\n    });\n    const cache = yield* Cache.makeWith((_: void) => options.acquire, {\n      capacity: 1,\n      timeToLive: (exit) =>\n        Exit.isSuccess(exit)\n          ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor)\n          : Duration.zero,\n    });\n\n    return {\n      get: Cache.get(cache, undefined).pipe(Effect.map((token) => token.value)),\n      invalidate: Cache.invalidate(cache, undefined),\n    };\n  });\n","export * from \"./token-cache\";\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgCA,MAAa,kBAAwB,YAInC,OAAO,IAAI,aAAa;CACtB,MAAM,gBAAgB,OAAO,MAAM,QAAQ,iBAAiB,IAAK;EAC/D,SAAS;EACT,SAAS;EACV,CAAC;CACF,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAY,QAAQ,SAAS;EAChE,UAAU;EACV,aAAa,SACX,KAAK,UAAU,KAAK,GAChB,SAAS,OAAO,SAAS,SAAS,KAAK,MAAM,UAAU,GAAG,cAAc,GACxE,SAAS;EAChB,CAAC;AAEF,QAAO;EACL,KAAK,MAAM,IAAI,OAAO,OAAU,CAAC,KAAK,OAAO,KAAK,UAAU,MAAM,MAAM,CAAC;EACzE,YAAY,MAAM,WAAW,OAAO,OAAU;EAC/C;EACD"}

package/dist/connector-app/index.d.ts

@@ -1,9 +1,14 @@
-import "../index-DyuT2jzP.js";
-import "../types-ZNrA-Dn0.js";
-import { i as start, n as App, r as StartOptions } from "../index-BlhJCpEk.js";
-import "../service-B7g_lHMI.js";
-import "../index-DaNW63Mv.js";
-import "../index-8KnVpoZp.js";
-import "../index-DK9YcR6W.js";
-import "../index-CQJsFJs3.js";
-export { App, StartOptions, start };
+import "../index-CTHVudvQ.js";
+import "../index-D54ySAKn.js";
+import "../types-B35J5YPu.js";
+import { d as CheckResult, f as ResourceCheckResult, i as start, n as App, p as check, r as StartOptions, u as CheckOptions } from "../index-DtJX_tic.js";
+import "../service-IE5qaf9x.js";
+import "../index-Coh_SkBs.js";
+import "../index-J42AZaXU.js";
+import "../index-CBT-JX-T.js";
+import "../metrics-CVAk1ALR.js";
+import "../index-Byr0Lf0C.js";
+import "../index-o1-Z3n7V.js";
+import "../index-CTLYQ74P.js";
+import "../index-fIIBVAai.js";
+export { App, CheckOptions, CheckResult, ResourceCheckResult, StartOptions, check, start };

package/dist/connector-app/index.js

@@ -1,8 +1,11 @@
 import "../errors-DHWylqRZ.js";
-import "../telemetry-p0dE5Mdu.js";
-import "../state-store-DG_4QpWG.js";
-import "../server-DAeFWT4o.js";
-import "../ingestion-CMCroK7Y.js";
-import { n as start } from "../connector-app-CvqbfN6R.js";
+import { n as start, r as check } from "../connector-app-d7Z-SZLQ.js";
+import "../metrics-CWPy2Aqc.js";
+import "../constants-BRhIX_rz.js";
+import "../service-DuFiQwid.js";
+import "../server-CXTQ7odl.js";
+import "../state-store-yl6LiTQ0.js";
+import "../status-B-oZBc33.js";
+import "../ingestion-BQo4cbGw.js";
 
-export { start };
+export { check, start };

package/dist/connector-app-d7Z-SZLQ.js

@@ -0,0 +1,64 @@
+import { t as __export } from "./chunk-Bp6m_JJh.js";
+import { n as run } from "./ingestion-BQo4cbGw.js";
+import { Context, DateTime, Effect, Layer, Tuple } from "effect";
+import { NodeHttpServer } from "@effect/platform-node";
+import { HttpRouter } from "effect/unstable/http";
+import { createServer } from "node:http";
+
+//#region src/connector-app/check.ts
+const ok = { _tag: "ok" };
+const error = (message) => ({
+	_tag: "error",
+	message
+});
+const fromEntries = (entries) => Object.fromEntries(entries);
+const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
+/**
+* Builds the connector service from its layer and validates only the selected
+* resources. The layer remains open until every check finishes.
+*/
+function check(service, layer, options) {
+	if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
+	return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
+		const resource = connector.resources.find((candidate) => candidate.name === name);
+		return (resource ? resource.check.pipe(Effect.match({
+			onFailure: (failure) => error(failure.message),
+			onSuccess: () => ok
+		})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
+	}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
+		onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
+		onSuccess: (results) => results
+	}));
+}
+
+//#endregion
+//#region src/connector-app/start.ts
+const start = (app, options) => Effect.gen(function* () {
+	const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+	const routes = app.webhooks?.map((route) => route.path) ?? [];
+	yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
+		port: options.port,
+		routes
+	}));
+	return yield* run(app, {
+		initialCutoff,
+		webhook: {
+			routes: app.webhooks ?? [],
+			healthPath: options.healthPath ?? "/health",
+			metricsPath: options.metricsPath,
+			statusPath: options.statusPath,
+			disableHttpLogger: options.disableHttpLogger ?? true
+		}
+	}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
+});
+
+//#endregion
+//#region src/connector-app/index.ts
+var connector_app_exports = /* @__PURE__ */ __export({
+	check: () => check,
+	start: () => start
+});
+
+//#endregion
+export { start as n, check as r, connector_app_exports as t };
+//# sourceMappingURL=connector-app-d7Z-SZLQ.js.map

package/dist/connector-app-d7Z-SZLQ.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"connector-app-d7Z-SZLQ.js","names":["ok: ResourceCheckResult","runIngestion"],"sources":["../src/connector-app/check.ts","../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n  | { readonly _tag: \"ok\" }\n  | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n  readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n  readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n  Resources extends ReadonlyArray<ResourceDefinition>,\n  Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n  fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n  Identifier,\n  const Resources extends ReadonlyArray<ResourceDefinition>,\n  E extends CheckFailure,\n  R,\n  const Name extends ResourceName<Resources>,\n>(\n  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n  layer: Layer.Layer<Identifier, E, R>,\n  options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n  if (options.resources.length === 0) {\n    return Effect.succeed(fromEntries<Name>([]));\n  }\n\n  return service.pipe(\n    Effect.flatMap((connector) =>\n      Effect.forEach(options.resources, (name) => {\n        const resource = connector.resources.find((candidate) => candidate.name === name);\n        const result = resource\n          ? resource.check.pipe(\n              Effect.match({\n                onFailure: (failure) => error(failure.message),\n                onSuccess: () => ok,\n              }),\n            )\n          : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n        return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n      }).pipe(Effect.map(fromEntries)),\n    ),\n    Effect.provide(layer),\n    Effect.match({\n      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n      onSuccess: (results) => results,\n    }),\n  );\n}\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/unstable/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n  readonly port: number;\n  readonly initialCutoff?: Cursor.Value;\n  readonly healthPath?: HttpRouter.PathInput;\n  readonly metricsPath?: HttpRouter.PathInput | false;\n  readonly statusPath?: HttpRouter.PathInput | false;\n  readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n  Effect.gen(function* () {\n    const initialCutoff =\n      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n    yield* Effect.logInfo(\"webhook server ready\").pipe(\n      Effect.annotateLogs({ port: options.port, routes }),\n    );\n\n    return yield* runIngestion(app, {\n      initialCutoff,\n      webhook: {\n        routes: app.webhooks ?? [],\n        healthPath: options.healthPath ?? \"/health\",\n        metricsPath: options.metricsPath,\n        statusPath: options.statusPath,\n        disableHttpLogger: options.disableHttpLogger ?? true,\n      },\n    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n  });\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;AAuBA,MAAMA,KAA0B,EAAE,MAAM,MAAM;AAE9C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;CAAS;AAEnF,MAAM,eACJ,YAC+B,OAAO,YAAY,QAAQ;AAE5D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;;;;;AAM5D,SAAgB,MAOd,SACA,OACA,SACuD;AACvD,KAAI,QAAQ,UAAU,WAAW,EAC/B,QAAO,OAAO,QAAQ,YAAkB,EAAE,CAAC,CAAC;AAG9C,QAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,KAAK;AAUjF,UATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,QAAQ;GAC9C,iBAAiB;GAClB,CAAC,CACH,GACD,OAAO,QAAQ,MAAM,+BAA+B,OAAO,CAAC,EAElD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;GACpE,CAAC,KAAK,OAAO,IAAI,YAAY,CAAC,CACjC,EACD,OAAO,QAAQ,MAAM,EACrB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,QAAQ,CAAC;EAC7E,YAAY,YAAY;EACzB,CAAC,CACH;;;;;ACrDH,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;CACpF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,KAAK,IAAI,EAAE;AAE7D,QAAO,OAAO,QAAQ,uBAAuB,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;EAAQ,CAAC,CACpD;AAED,QAAO,OAAOC,IAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,EAAE;GAC1B,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;GACjD;EACF,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,CAAC;EACnF"}

package/dist/constants-BRhIX_rz.js

@@ -0,0 +1,27 @@
+import { HashSet } from "effect";
+
+//#region src/runtime-config/constants.ts
+/** Shared runtime keys owned by Airfoil rather than an individual connector. */
+const PlatformRuntimeKey = {
+	configPath: "AIRFOIL_CONFIG_PATH",
+	connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
+	stateTable: "AIRFOIL_STATE_TABLE",
+	postgresConnectionString: "POSTGRES_CONNECTION_STRING",
+	wingsHost: "WINGS_HOST",
+	wingsNamespace: "WINGS_NAMESPACE",
+	otelEnabled: "OTEL_ENABLED",
+	otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
+	otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",
+	otelServiceName: "OTEL_SERVICE_NAME",
+	otelServiceVersion: "OTEL_SERVICE_VERSION",
+	otelResourceAttributes: "OTEL_RESOURCE_ATTRIBUTES"
+};
+/** Defaults for optional platform-owned runtime configuration. */
+const PlatformRuntimeDefault = { stateTable: "_airfoil_connectors_state" };
+const platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));
+/** @internal */
+const isPlatformRuntimeKey = (key) => HashSet.has(platformRuntimeKeys, key);
+
+//#endregion
+export { PlatformRuntimeKey as n, isPlatformRuntimeKey as r, PlatformRuntimeDefault as t };
+//# sourceMappingURL=constants-BRhIX_rz.js.map

package/dist/constants-BRhIX_rz.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants-BRhIX_rz.js","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n  configPath: \"AIRFOIL_CONFIG_PATH\",\n  connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n  stateTable: \"AIRFOIL_STATE_TABLE\",\n  postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n  wingsHost: \"WINGS_HOST\",\n  wingsNamespace: \"WINGS_NAMESPACE\",\n  otelEnabled: \"OTEL_ENABLED\",\n  otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n  otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n  otelServiceName: \"OTEL_SERVICE_NAME\",\n  otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n  otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n  stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;CACzB;;AAGD,MAAa,yBAAyB,EACpC,YAAY,6BACb;AAED,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,mBAAmB,CAAC;;AAGnF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,IAAI"}

package/dist/errors/index.d.ts

@@ -1,2 +1,2 @@
-import { t as ConnectorError } from "../index-DyuT2jzP.js";
+import { t as ConnectorError } from "../index-D54ySAKn.js";
 export { ConnectorError };