@useairfoil/connector-kit 0.5.3 → 0.6.1

package/src/state-store/layer-memory.ts

@@ -1,5 +1,5 @@
 import { Layer } from "effect";
-import { KeyValueStore } from "effect/unstable/persistence";
+import { KeyValueStore } from "effect/persistence";
 
 import type { StateStore } from "./service";
 

package/src/state-store/layer-sql.ts

@@ -1,6 +1,6 @@
 import { Config, Effect, Layer } from "effect";
-import { KeyValueStore } from "effect/unstable/persistence";
-import { SqlClient } from "effect/unstable/sql";
+import { KeyValueStore } from "effect/persistence";
+import { SqlClient } from "effect/sql";
 
 import type { StateStore } from "./service";
 
@@ -32,8 +32,8 @@ const scopedKeyValueStore = (connectorInstanceId: string) =>
 export const layerSql = (): Layer.Layer<StateStore, Config.ConfigError, SqlClient.SqlClient> =>
   Layer.unwrap(
     Config.all({
-      connectorInstanceId: Config.nonEmptyString(PlatformRuntimeKey.connectorInstanceId),
-      table: Config.nonEmptyString(PlatformRuntimeKey.stateTable).pipe(
+      connectorInstanceId: Config.NonEmptyString(PlatformRuntimeKey.connectorInstanceId),
+      table: Config.NonEmptyString(PlatformRuntimeKey.stateTable).pipe(
         Config.withDefault(PlatformRuntimeDefault.stateTable),
       ),
     }).pipe(

package/src/state-store/schema.ts

@@ -18,11 +18,11 @@ export const ChangesStateSchema = Schema.Struct({
 
 export const StateSourceSchema = Schema.Literals(["backfill", "changes"]);
 
-export const StateOperationSchema = Schema.Literals(["fetch", "publish", "checkpoint"]);
+export const StateOperationSchema = Schema.Literals(["fetch", "ingest", "checkpoint"]);
 
 export const StateErrorCodeSchema = Schema.Literals([
   "fetch_failed",
-  "publish_failed",
+  "ingest_failed",
   "checkpoint_failed",
 ]);
 

package/src/state-store/state.ts

@@ -1,16 +1,28 @@
-import type { Cursor, ResourceDefinition, ResourceState, SyncState } from "../core/types";
+import type {
+  ConnectorDefinition,
+  Cursor,
+  ResourceDefinition,
+  ResourceState,
+  SyncState,
+} from "../core/types";
 
 /** Normalizes a cursor value to its persisted/wire representation (never a Date). */
 export const normalizeCursor = (value: Cursor.Value): string | number =>
   value instanceof Date ? value.toISOString() : value;
 
+/** Whether a resource gets changes, from its own `changes` or the connector's feed. */
+export const hasChanges = (connector: ConnectorDefinition, resource: ResourceDefinition) =>
+  resource.changes !== undefined ||
+  (connector.changes?.resources.some((member) => member.name === resource.name) ?? false);
+
 /** Derives connector-facing sync state from durable state and resource capabilities. */
 export const deriveSyncState = (
+  connector: ConnectorDefinition,
   resource: ResourceDefinition,
   state: ResourceState | undefined,
 ): SyncState => {
   if (state?.lastError) return "error";
-  if (!resource.backfill && !resource.changes) return "live";
+  if (!resource.backfill && !hasChanges(connector, resource)) return "live";
   if (!state) return "pending";
   if (resource.backfill && (!state.backfill || !state.backfill.completed)) return "backfilling";
   return "live";

package/src/status/route.ts

@@ -1,5 +1,5 @@
 import { Cause, Effect } from "effect";
-import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
+import { HttpRouter, HttpServerResponse } from "effect/http";
 
 import type { ConnectorDefinition } from "../core/types";
 import type { ResourceStatus, StatusUnavailableResponse } from "./types";
@@ -24,7 +24,7 @@ export const statusRoute = (
             Effect.map(
               (state): ResourceStatus => ({
                 name: resource.name,
-                state: deriveSyncState(resource, state),
+                state: deriveSyncState(connector, resource, state),
                 ...(state?.backfill
                   ? {
                       backfill: {

package/src/telemetry.ts

@@ -1,6 +1,6 @@
 import { Cause, Config, Effect, Layer, Metric, Option, type Duration } from "effect";
-import { Headers } from "effect/unstable/http";
-import * as Observability from "effect/unstable/observability";
+import { Headers } from "effect/http";
+import * as Observability from "effect/observability";
 
 import { PlatformRuntimeKey } from "./runtime-config/constants";
 
@@ -9,7 +9,7 @@ export const SpanName = {
   batchProcess: "connector.batch.process",
   webhookDecode: "connector.webhook.decode",
   webhookHandle: "connector.webhook.handle",
-  publish: "connector.publish",
+  ingest: "connector.ingest",
   stateSet: "connector.state.set",
   apiFetch: "connector.api.fetch",
 } as const;
@@ -20,9 +20,13 @@ export const Attr = {
   connectorResourcesCount: "airfoil.connector.resources.count",
   resourceName: "airfoil.resource.name",
   resourceSource: "airfoil.resource.source",
-  batchMutations: "airfoil.batch.mutations",
+  batchRows: "airfoil.batch.rows",
+  batchOutcome: "airfoil.batch.outcome",
   webhookPath: "airfoil.webhook.path",
-  publisherSuccess: "airfoil.publisher.success",
+  webhookOutcome: "airfoil.webhook.outcome",
+  syncState: "airfoil.connector.sync.state",
+  apiRetryReason: "airfoil.connector.api.retry.reason",
+  ingestionSuccess: "airfoil.ingestion.success",
   stateKey: "airfoil.state.key",
   apiPath: "airfoil.api.path",
   errorPhase: "airfoil.error.phase",
@@ -31,6 +35,13 @@ export const Attr = {
   errorDetails: "airfoil.error.details",
 } as const;
 
+/** Resource attributes injected by the hosted operator through standard OTel environment keys. */
+export const ResourceAttr = {
+  serviceInstanceId: "service.instance.id",
+  connectorRevision: "airfoil.connector.revision",
+  teamId: "airfoil.team.id",
+} as const;
+
 /** Span event names for high-cardinality reproduction details. */
 export const EventName = {
   batchCheckpoint: "airfoil.batch.checkpoint",
@@ -47,9 +58,9 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
 const maxErrorMessageLength = 500;
 
 const OtlpEnvConfig = Config.all({
-  enabled: Config.boolean(PlatformRuntimeKey.otelEnabled).pipe(Config.withDefault(false)),
-  baseUrl: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpEndpoint)),
-  rawHeaders: Config.option(Config.string(PlatformRuntimeKey.otelExporterOtlpHeaders)),
+  enabled: Config.Boolean(PlatformRuntimeKey.otelEnabled).pipe(Config.withDefault(false)),
+  baseUrl: Config.option(Config.String(PlatformRuntimeKey.otelExporterOtlpEndpoint)),
+  rawHeaders: Config.option(Config.String(PlatformRuntimeKey.otelExporterOtlpHeaders)),
 });
 
 const defaultRedactedHeaders: ReadonlyArray<string | RegExp> = [
@@ -165,6 +176,7 @@ export type OtlpTracingOptions = {
   readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
 };
 
+/** OTLP metric export settings. */
 export type OtlpMetricsOptions = {
   readonly exportInterval?: Duration.Input;
 };
@@ -211,8 +223,8 @@ export const layer = (config: OtlpTracingConfig = {}, options: OtlpTracingOption
  *
  * @example
  * Telemetry.layerConfig({
- *   enabled: Config.boolean("MY_OTEL_ENABLED"),
- *   endpoint: Config.string("MY_COLLECTOR_URL"),
+ *   enabled: Config.Boolean("MY_OTEL_ENABLED"),
+ *   endpoint: Config.String("MY_COLLECTOR_URL"),
  * })
  */
 export const layerConfig = (
@@ -251,6 +263,7 @@ export const layerOtlpTracing = (options: OtlpTracingOptions = {}) =>
     }),
   );
 
+/** Exports connector metrics over OTLP when telemetry is enabled. */
 export const layerOtlpMetrics = (options: OtlpMetricsOptions = {}) =>
   Layer.unwrap(
     Effect.gen(function* () {
@@ -270,9 +283,11 @@ export const layerOtlpMetrics = (options: OtlpMetricsOptions = {}) =>
     }),
   );
 
+/** Enables OTLP traces and metrics with shared environment configuration. */
 export const layerOtlp = (options: OtlpTracingOptions & OtlpMetricsOptions = {}) =>
   Layer.mergeAll(layerOtlpTracing(options), layerOtlpMetrics(options));
 
+/** Periodically logs the in-memory metric snapshot for local debugging. */
 export const layerMetricsConsoleDump = (interval: Duration.Input = "30 seconds") =>
   Layer.effectDiscard(
     Effect.gen(function* () {

package/src/webhook/server.ts

@@ -1,18 +1,16 @@
 import { Cause, Context, Effect, Queue, Schema } from "effect";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
 
 import type {
   ResourceBatch,
   ResourceDefinition,
-  ResourceMutation,
-  WebhookHandler,
   WebhookRoute,
   WebhookRouteContext,
 } from "../core/types";
 
 import { ConnectorError } from "../errors";
+import { ingestBatch } from "../ingestor/instrumented";
 import * as Metrics from "../metrics";
-import { publishBatch } from "../publisher/instrumented";
 import { Attr, SpanName, annotateError } from "../telemetry";
 
 export type QueuedWebhookBatch = {
@@ -27,39 +25,23 @@ export class WebhookQueue extends Context.Service<
   }
 >()("@useairfoil/connector-kit/WebhookQueue") {}
 
-const resourceBatch = <Row extends object>(
-  mutations: ReadonlyArray<ResourceMutation<Row>>,
-): ResourceBatch<Row> => ({
-  mutations,
+const resourceBatch = <Row extends object>(rows: ReadonlyArray<Row>): ResourceBatch<Row> => ({
+  rows,
 });
 
-const decodeAndHandleWebhook = <Row extends object, Payload>(
-  resourceName: string,
-  webhook: WebhookHandler<Row, Payload>,
-  payload: unknown,
-) =>
-  Schema.decodeUnknownEffect(webhook.schema)(payload).pipe(
-    Effect.mapError(
-      (cause) =>
-        new ConnectorError({
-          message: `Invalid webhook payload for resource ${resourceName}`,
-          cause,
-        }),
-    ),
-    Effect.flatMap((decoded) => webhook.handler({ payload: decoded })),
-  );
-
 const makeRouteContext = (
   request: HttpServerRequest.HttpServerRequest,
   rawBody: Uint8Array,
   payload: unknown,
   batches: Array<QueuedWebhookBatch>,
+  resourceNames: ReadonlyArray<string>,
 ): WebhookRouteContext<ReadonlyArray<ResourceDefinition>, unknown> => ({
   request,
   rawBody,
   payload,
   to: (resource, payload) =>
     Effect.gen(function* () {
+      if (!resourceNames.includes(resource.name)) return;
       if (!resource.webhook) {
         return yield* Effect.fail(
           new ConnectorError({
@@ -68,27 +50,28 @@ const makeRouteContext = (
         );
       }
 
-      const mutations = yield* decodeAndHandleWebhook(resource.name, resource.webhook, payload);
+      // The route already decoded the payload, so don't decode it again.
+      const rows = yield* resource.webhook.handler({ payload });
       batches.push({
         resource: resource.name,
-        batch: resourceBatch(mutations),
+        batch: resourceBatch(rows),
       });
     }),
 });
 
 const compactBatches = (batches: ReadonlyArray<QueuedWebhookBatch>) => {
-  const grouped = new Map<string, ResourceMutation[]>();
+  const grouped = new Map<string, object[]>();
   for (const item of batches) {
     const current = grouped.get(item.resource) ?? [];
-    grouped.set(item.resource, [...current, ...item.batch.mutations]);
+    grouped.set(item.resource, [...current, ...item.batch.rows]);
   }
-  return Array.from(grouped.entries()).map(([resource, mutations]) => ({
+  return Array.from(grouped.entries()).map(([resource, rows]) => ({
     resource,
-    batch: resourceBatch(mutations),
+    batch: resourceBatch(rows),
   }));
 };
 
-const publishBatches = (
+const ingestBatches = (
   batches: ReadonlyArray<QueuedWebhookBatch>,
   options: { readonly connectorName: string },
 ) =>
@@ -96,7 +79,7 @@ const publishBatches = (
     yield* Effect.forEach(
       batches,
       (batch) =>
-        publishBatch({
+        ingestBatch({
           connector: options.connectorName,
           resource: batch.resource,
           source: "webhook",
@@ -118,7 +101,13 @@ const parseJson = (rawBody: Uint8Array) =>
     catch: (cause) => new ConnectorError({ message: "Invalid webhook JSON", cause }),
   });
 
-const makeHandler = (route: WebhookRoute, options: { readonly connectorName: string }) =>
+type RouterOptions = {
+  readonly connectorName: string;
+  /** Events for other resources are skipped. */
+  readonly resourceNames: ReadonlyArray<string>;
+};
+
+const makeHandler = (route: WebhookRoute, options: RouterOptions) =>
   Effect.gen(function* () {
     const request = yield* HttpServerRequest.HttpServerRequest;
     const collected: Array<QueuedWebhookBatch> = [];
@@ -129,6 +118,7 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
         onSuccess: (value) => ({ _tag: "Success" as const, value }),
       }),
     );
+
     if (rawBodyResult._tag === "Error") {
       yield* Metrics.recordWebhookRequest({
         connector: options.connectorName,
@@ -144,6 +134,7 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
         onSuccess: (value) => ({ _tag: "Success" as const, value }),
       }),
     );
+
     if (jsonResult._tag === "Error") {
       yield* Metrics.recordWebhookRequest({
         connector: options.connectorName,
@@ -159,6 +150,7 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
         onSuccess: (value) => ({ _tag: "Success" as const, value }),
       }),
     );
+
     if (payloadResult._tag === "Error") {
       yield* Metrics.recordWebhookRequest({
         connector: options.connectorName,
@@ -170,7 +162,15 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
 
     const response = yield* Effect.withSpan(
       route
-        .handler(makeRouteContext(request, rawBodyResult.value, payloadResult.value, collected))
+        .handler(
+          makeRouteContext(
+            request,
+            rawBodyResult.value,
+            payloadResult.value,
+            collected,
+            options.resourceNames,
+          ),
+        )
         .pipe(Effect.tapError((error) => annotateError("webhook_handle", error))),
       SpanName.webhookHandle,
       { attributes: { [Attr.webhookPath]: route.path } },
@@ -188,12 +188,13 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
         path: String(route.path),
         outcome: response.status >= 400 ? "rejected" : "ok",
       });
+
       return response;
     }
 
     yield* Effect.withSpan(
-      publishBatches(batches, { connectorName: options.connectorName }).pipe(
-        Effect.tapError((error) => annotateError("webhook_publish", error)),
+      ingestBatches(batches, { connectorName: options.connectorName }).pipe(
+        Effect.tapError((error) => annotateError("webhook_ingest", error)),
       ),
       SpanName.webhookHandle,
       { attributes: { [Attr.webhookPath]: route.path } },
@@ -204,6 +205,7 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
       path: String(route.path),
       outcome: response.status >= 400 ? "rejected" : "ok",
     });
+
     return response;
   }).pipe(
     Effect.catchCause((cause) =>
@@ -225,10 +227,7 @@ const makeHandler = (route: WebhookRoute, options: { readonly connectorName: str
     ),
   );
 
-export const router = (
-  routes: ReadonlyArray<WebhookRoute>,
-  options: { readonly connectorName: string },
-) =>
+export const router = (routes: ReadonlyArray<WebhookRoute>, options: RouterOptions) =>
   HttpRouter.addAll(
     routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route, options))),
   );

package/dist/auth/index.d.ts

@@ -1,2 +0,0 @@
-import { i as makeTokenCache, n as ExpiringToken, r as TokenCache } from "../index-CTHVudvQ.js";
-export { ExpiringToken, TokenCache, makeTokenCache };

package/dist/auth/index.js

@@ -1,3 +0,0 @@
-import { n as makeTokenCache } from "../auth-DvU_CZ7C.js";
-
-export { makeTokenCache };

package/dist/auth-DvU_CZ7C.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"auth-DvU_CZ7C.js","names":[],"sources":["../src/auth/token-cache.ts","../src/auth/index.ts"],"sourcesContent":["import { Cache, Duration, Effect, Exit, Number, Redacted } from \"effect\";\n\nexport interface ExpiringToken {\n  /** The credential kept in memory by the cache. */\n  readonly value: Redacted.Redacted<string>;\n  /** The lifetime reported by the provider. */\n  readonly expiresIn: Duration.Duration;\n}\n\nexport interface TokenCache<E> {\n  /** Returns the cached token or performs one shared acquisition. */\n  readonly get: Effect.Effect<Redacted.Redacted<string>, E>;\n  /** Removes the cached token so the next `get` reacquires it. */\n  readonly invalidate: Effect.Effect<void>;\n}\n\n/**\n * Creates a lazy, in-memory token cache around an acquisition effect.\n *\n * `refreshFactor` controls how much of the provider lifetime is cached. It is\n * clamped between `0` and `1`.\n *\n * @example\n * ```ts\n * const cache = yield* makeTokenCache({\n *   acquire: exchangeCredentials,\n *   refreshFactor: 0.5,\n * })\n *\n * const token = yield* cache.get\n * ```\n */\nexport const makeTokenCache = <E, R>(options: {\n  readonly acquire: Effect.Effect<ExpiringToken, E, R>;\n  readonly refreshFactor?: number;\n}): Effect.Effect<TokenCache<E>, never, R> =>\n  Effect.gen(function* () {\n    const refreshFactor = Number.clamp(options.refreshFactor ?? 0.5, {\n      minimum: 0,\n      maximum: 1,\n    });\n    const cache = yield* Cache.makeWith((_: void) => options.acquire, {\n      capacity: 1,\n      timeToLive: (exit) =>\n        Exit.isSuccess(exit)\n          ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor)\n          : Duration.zero,\n    });\n\n    return {\n      get: Cache.get(cache, undefined).pipe(Effect.map((token) => token.value)),\n      invalidate: Cache.invalidate(cache, undefined),\n    };\n  });\n","export * from \"./token-cache\";\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgCA,MAAa,kBAAwB,YAInC,OAAO,IAAI,aAAa;CACtB,MAAM,gBAAgB,OAAO,MAAM,QAAQ,iBAAiB,IAAK;EAC/D,SAAS;EACT,SAAS;EACV,CAAC;CACF,MAAM,QAAQ,OAAO,MAAM,UAAU,MAAY,QAAQ,SAAS;EAChE,UAAU;EACV,aAAa,SACX,KAAK,UAAU,KAAK,GAChB,SAAS,OAAO,SAAS,SAAS,KAAK,MAAM,UAAU,GAAG,cAAc,GACxE,SAAS;EAChB,CAAC;AAEF,QAAO;EACL,KAAK,MAAM,IAAI,OAAO,OAAU,CAAC,KAAK,OAAO,KAAK,UAAU,MAAM,MAAM,CAAC;EACzE,YAAY,MAAM,WAAW,OAAO,OAAU;EAC/C;EACD"}

package/dist/connector-app/index.d.ts

@@ -1,14 +0,0 @@
-import "../index-CTHVudvQ.js";
-import "../index-D54ySAKn.js";
-import "../types-B35J5YPu.js";
-import { d as CheckResult, f as ResourceCheckResult, i as start, n as App, p as check, r as StartOptions, u as CheckOptions } from "../index-DtJX_tic.js";
-import "../service-IE5qaf9x.js";
-import "../index-Coh_SkBs.js";
-import "../index-J42AZaXU.js";
-import "../index-CBT-JX-T.js";
-import "../metrics-CVAk1ALR.js";
-import "../index-Byr0Lf0C.js";
-import "../index-o1-Z3n7V.js";
-import "../index-Cq2IzY7T.js";
-import "../index-DKWIh6M1.js";
-export { App, CheckOptions, CheckResult, ResourceCheckResult, StartOptions, check, start };

package/dist/connector-app/index.js

@@ -1,11 +0,0 @@
-import "../errors-DHWylqRZ.js";
-import { n as start, r as check } from "../connector-app-d7Z-SZLQ.js";
-import "../metrics-CWPy2Aqc.js";
-import "../constants-BRhIX_rz.js";
-import "../service-DuFiQwid.js";
-import "../server-CXTQ7odl.js";
-import "../state-store-yl6LiTQ0.js";
-import "../status-B-oZBc33.js";
-import "../ingestion-BQo4cbGw.js";
-
-export { check, start };

package/dist/connector-app-d7Z-SZLQ.js

@@ -1,64 +0,0 @@
-import { t as __export } from "./chunk-Bp6m_JJh.js";
-import { n as run } from "./ingestion-BQo4cbGw.js";
-import { Context, DateTime, Effect, Layer, Tuple } from "effect";
-import { NodeHttpServer } from "@effect/platform-node";
-import { HttpRouter } from "effect/unstable/http";
-import { createServer } from "node:http";
-
-//#region src/connector-app/check.ts
-const ok = { _tag: "ok" };
-const error = (message) => ({
-	_tag: "error",
-	message
-});
-const fromEntries = (entries) => Object.fromEntries(entries);
-const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
-/**
-* Builds the connector service from its layer and validates only the selected
-* resources. The layer remains open until every check finishes.
-*/
-function check(service, layer, options) {
-	if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
-	return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
-		const resource = connector.resources.find((candidate) => candidate.name === name);
-		return (resource ? resource.check.pipe(Effect.match({
-			onFailure: (failure) => error(failure.message),
-			onSuccess: () => ok
-		})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
-	}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
-		onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
-		onSuccess: (results) => results
-	}));
-}
-
-//#endregion
-//#region src/connector-app/start.ts
-const start = (app, options) => Effect.gen(function* () {
-	const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
-	const routes = app.webhooks?.map((route) => route.path) ?? [];
-	yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
-		port: options.port,
-		routes
-	}));
-	return yield* run(app, {
-		initialCutoff,
-		webhook: {
-			routes: app.webhooks ?? [],
-			healthPath: options.healthPath ?? "/health",
-			metricsPath: options.metricsPath,
-			statusPath: options.statusPath,
-			disableHttpLogger: options.disableHttpLogger ?? true
-		}
-	}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
-});
-
-//#endregion
-//#region src/connector-app/index.ts
-var connector_app_exports = /* @__PURE__ */ __export({
-	check: () => check,
-	start: () => start
-});
-
-//#endregion
-export { start as n, check as r, connector_app_exports as t };
-//# sourceMappingURL=connector-app-d7Z-SZLQ.js.map

package/dist/connector-app-d7Z-SZLQ.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"connector-app-d7Z-SZLQ.js","names":["ok: ResourceCheckResult","runIngestion"],"sources":["../src/connector-app/check.ts","../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n  | { readonly _tag: \"ok\" }\n  | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n  readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n  readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n  Resources extends ReadonlyArray<ResourceDefinition>,\n  Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n  fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n  Identifier,\n  const Resources extends ReadonlyArray<ResourceDefinition>,\n  E extends CheckFailure,\n  R,\n  const Name extends ResourceName<Resources>,\n>(\n  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n  layer: Layer.Layer<Identifier, E, R>,\n  options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n  if (options.resources.length === 0) {\n    return Effect.succeed(fromEntries<Name>([]));\n  }\n\n  return service.pipe(\n    Effect.flatMap((connector) =>\n      Effect.forEach(options.resources, (name) => {\n        const resource = connector.resources.find((candidate) => candidate.name === name);\n        const result = resource\n          ? resource.check.pipe(\n              Effect.match({\n                onFailure: (failure) => error(failure.message),\n                onSuccess: () => ok,\n              }),\n            )\n          : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n        return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n      }).pipe(Effect.map(fromEntries)),\n    ),\n    Effect.provide(layer),\n    Effect.match({\n      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n      onSuccess: (results) => results,\n    }),\n  );\n}\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/unstable/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n  readonly port: number;\n  readonly initialCutoff?: Cursor.Value;\n  readonly healthPath?: HttpRouter.PathInput;\n  readonly metricsPath?: HttpRouter.PathInput | false;\n  readonly statusPath?: HttpRouter.PathInput | false;\n  readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n  Effect.gen(function* () {\n    const initialCutoff =\n      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n    yield* Effect.logInfo(\"webhook server ready\").pipe(\n      Effect.annotateLogs({ port: options.port, routes }),\n    );\n\n    return yield* runIngestion(app, {\n      initialCutoff,\n      webhook: {\n        routes: app.webhooks ?? [],\n        healthPath: options.healthPath ?? \"/health\",\n        metricsPath: options.metricsPath,\n        statusPath: options.statusPath,\n        disableHttpLogger: options.disableHttpLogger ?? true,\n      },\n    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n  });\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;AAuBA,MAAMA,KAA0B,EAAE,MAAM,MAAM;AAE9C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;CAAS;AAEnF,MAAM,eACJ,YAC+B,OAAO,YAAY,QAAQ;AAE5D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;;;;;AAM5D,SAAgB,MAOd,SACA,OACA,SACuD;AACvD,KAAI,QAAQ,UAAU,WAAW,EAC/B,QAAO,OAAO,QAAQ,YAAkB,EAAE,CAAC,CAAC;AAG9C,QAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,KAAK;AAUjF,UATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,QAAQ;GAC9C,iBAAiB;GAClB,CAAC,CACH,GACD,OAAO,QAAQ,MAAM,+BAA+B,OAAO,CAAC,EAElD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;GACpE,CAAC,KAAK,OAAO,IAAI,YAAY,CAAC,CACjC,EACD,OAAO,QAAQ,MAAM,EACrB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,QAAQ,CAAC;EAC7E,YAAY,YAAY;EACzB,CAAC,CACH;;;;;ACrDH,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;CACpF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,KAAK,IAAI,EAAE;AAE7D,QAAO,OAAO,QAAQ,uBAAuB,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;EAAQ,CAAC,CACpD;AAED,QAAO,OAAOC,IAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,EAAE;GAC1B,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;GACjD;EACF,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,CAAC;EACnF"}

package/dist/constants-BRhIX_rz.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants-BRhIX_rz.js","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n  configPath: \"AIRFOIL_CONFIG_PATH\",\n  connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n  stateTable: \"AIRFOIL_STATE_TABLE\",\n  postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n  wingsHost: \"WINGS_HOST\",\n  wingsNamespace: \"WINGS_NAMESPACE\",\n  otelEnabled: \"OTEL_ENABLED\",\n  otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n  otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n  otelServiceName: \"OTEL_SERVICE_NAME\",\n  otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n  otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n  stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;CACzB;;AAGD,MAAa,yBAAyB,EACpC,YAAY,6BACb;AAED,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,mBAAmB,CAAC;;AAGnF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,IAAI"}

package/dist/errors/index.d.ts

@@ -1,2 +0,0 @@
-import { t as ConnectorError } from "../index-D54ySAKn.js";
-export { ConnectorError };

package/dist/errors/index.js

@@ -1,3 +0,0 @@
-import { t as ConnectorError } from "../errors-DHWylqRZ.js";
-
-export { ConnectorError };

package/dist/errors-DHWylqRZ.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"errors-DHWylqRZ.js","names":[],"sources":["../src/errors/connector-error.ts"],"sourcesContent":["import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n"],"mappings":";;;AAEA,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE"}

package/dist/index-Byr0Lf0C.d.ts

@@ -1,25 +0,0 @@
-import { t as ConnectorError } from "./index-D54ySAKn.js";
-import { c as ResourceBatch, n as ConnectorDefinition } from "./types-B35J5YPu.js";
-import { a as PublisherService, i as Publisher, n as PublishOptions, r as PublishSource, t as PublishAck } from "./service-IE5qaf9x.js";
-import { Layer } from "effect";
-import * as Wings from "@useairfoil/wings";
-
-//#region src/publisher/wings.d.ts
-type WingsTableMapping = {
-  readonly name: string;
-  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
-};
-type WingsPublisherConfig = {
-  readonly connector: ConnectorDefinition;
-  readonly tables: Record<string, string | WingsTableMapping>;
-};
-declare const layerWings: (config: WingsPublisherConfig) => Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient>;
-//#endregion
-//#region src/publisher/console.d.ts
-declare const layerConsole: Layer.Layer<Publisher, never, never>;
-declare namespace index_d_exports {
-  export { PublishAck, PublishOptions, PublishSource, Publisher, PublisherService, ResourceBatch, WingsPublisherConfig, layerConsole, layerWings };
-}
-//#endregion
-export { layerWings as i, layerConsole as n, WingsPublisherConfig as r, index_d_exports as t };
-//# sourceMappingURL=index-Byr0Lf0C.d.ts.map

package/dist/index-o1-Z3n7V.d.ts

@@ -1,55 +0,0 @@
-import { FileSystem } from "effect";
-import * as effect_Types0 from "effect/Types";
-import * as effect_Cause0 from "effect/Cause";
-import * as effect_Layer0 from "effect/Layer";
-
-//#region src/runtime-config/error.d.ts
-type RuntimeConfigErrorCode = "CONFIG_PATH_MISSING" | "CONFIG_FILE_UNREADABLE" | "CONFIG_JSON_INVALID";
-declare const RuntimeConfigError_base: new <A extends Record<string, any> = {}>(args: effect_Types0.VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => effect_Cause0.YieldableError & {
-  readonly _tag: "RuntimeConfigError";
-} & Readonly<A>;
-/** Expected hosted-config failure with a stable code and diagnostic cause. */
-declare class RuntimeConfigError extends RuntimeConfigError_base<{
-  readonly code: RuntimeConfigErrorCode;
-  readonly message: string;
-  readonly cause: Error;
-}> {}
-//#endregion
-//#region src/runtime-config/constants.d.ts
-/** Shared runtime keys owned by Airfoil rather than an individual connector. */
-declare const PlatformRuntimeKey: {
-  readonly configPath: "AIRFOIL_CONFIG_PATH";
-  readonly connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID";
-  readonly stateTable: "AIRFOIL_STATE_TABLE";
-  readonly postgresConnectionString: "POSTGRES_CONNECTION_STRING";
-  readonly wingsHost: "WINGS_HOST";
-  readonly wingsNamespace: "WINGS_NAMESPACE";
-  readonly otelEnabled: "OTEL_ENABLED";
-  readonly otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT";
-  readonly otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS";
-  readonly otelServiceName: "OTEL_SERVICE_NAME";
-  readonly otelServiceVersion: "OTEL_SERVICE_VERSION";
-  readonly otelResourceAttributes: "OTEL_RESOURCE_ATTRIBUTES";
-};
-/** Defaults for optional platform-owned runtime configuration. */
-declare const PlatformRuntimeDefault: {
-  readonly stateTable: "_airfoil_connectors_state";
-};
-//#endregion
-//#region src/runtime-config/layer.d.ts
-/**
- * Loads the required JSON file and lets environment values override it per key.
- * Connector field validation remains in the connector's Effect Config.
- *
- * @example
- * ```ts
- * program.pipe(Effect.provide(layerHosted()));
- * ```
- */
-declare const layerHosted: () => effect_Layer0.Layer<never, RuntimeConfigError, FileSystem.FileSystem>;
-declare namespace index_d_exports {
-  export { PlatformRuntimeDefault, PlatformRuntimeKey, RuntimeConfigError, RuntimeConfigErrorCode, layerHosted };
-}
-//#endregion
-export { RuntimeConfigError as a, PlatformRuntimeKey as i, layerHosted as n, RuntimeConfigErrorCode as o, PlatformRuntimeDefault as r, index_d_exports as t };
-//# sourceMappingURL=index-o1-Z3n7V.d.ts.map

package/dist/index.d.ts

@@ -1,14 +0,0 @@
-import { t as index_d_exports } from "./index-CTHVudvQ.js";
-import { t as ConnectorError } from "./index-D54ySAKn.js";
-import { S as WebhookRouteContext, _ as ResourceState, a as FetchChangesResult, b as WebhookHandler, c as ResourceBatch, d as ResourceMutation, g as ResourceSchema, h as ResourceRows, l as ResourceDefinition, m as ResourceRow, n as ConnectorDefinition, o as FetchPageResult, p as ResourcePayload, r as Cursor$1, s as PageFetch, t as ChangesFetch, u as ResourceField, x as WebhookRoute, y as WebhookAckMode } from "./types-B35J5YPu.js";
-import { a as telemetry_d_exports, c as Fetch, l as Resource, o as Connector, s as Cursor, t as index_d_exports$1 } from "./index-DtJX_tic.js";
-import "./service-IE5qaf9x.js";
-import { t as index_d_exports$6 } from "./index-Coh_SkBs.js";
-import { t as index_d_exports$2 } from "./index-J42AZaXU.js";
-import { t as index_d_exports$3 } from "./index-CBT-JX-T.js";
-import { g as metrics_d_exports } from "./metrics-CVAk1ALR.js";
-import { t as index_d_exports$4 } from "./index-Byr0Lf0C.js";
-import { t as index_d_exports$5 } from "./index-o1-Z3n7V.js";
-import { t as index_d_exports$7 } from "./index-Cq2IzY7T.js";
-import { t as index_d_exports$8 } from "./index-DKWIh6M1.js";
-export { index_d_exports as Auth, ChangesFetch, Connector, index_d_exports$1 as ConnectorApp, ConnectorDefinition, ConnectorError, Cursor, Cursor$1 as CursorTypes, Fetch, FetchChangesResult, FetchPageResult, index_d_exports$2 as Ingestion, index_d_exports$3 as Manifest, metrics_d_exports as Metrics, PageFetch, index_d_exports$4 as Publisher, Resource, ResourceBatch, ResourceDefinition, ResourceField, ResourceMutation, ResourcePayload, ResourceRow, ResourceRows, ResourceSchema, ResourceState, index_d_exports$5 as RuntimeConfig, index_d_exports$6 as StateStore, index_d_exports$7 as Status, telemetry_d_exports as Telemetry, index_d_exports$8 as Webhook, WebhookAckMode, WebhookHandler, WebhookRoute, WebhookRouteContext };