@useairfoil/connector-kit 0.5.0 → 0.5.1

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@useairfoil/connector-kit",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "repository": "useairfoil/typescript-sdk",
   "files": [
     "dist",
     "src",
+    "METRICS.md",
     "README.md"
   ],
   "type": "module",
@@ -16,6 +17,11 @@
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     },
+    "./auth": {
+      "types": "./dist/auth/index.d.ts",
+      "import": "./dist/auth/index.js",
+      "default": "./dist/auth/index.js"
+    },
     "./connector-app": {
       "types": "./dist/connector-app/index.d.ts",
       "import": "./dist/connector-app/index.js",
@@ -26,16 +32,36 @@
       "import": "./dist/ingestion/index.js",
       "default": "./dist/ingestion/index.js"
     },
+    "./manifest": {
+      "types": "./dist/manifest/index.d.ts",
+      "import": "./dist/manifest/index.js",
+      "default": "./dist/manifest/index.js"
+    },
+    "./metrics": {
+      "types": "./dist/metrics.d.ts",
+      "import": "./dist/metrics.js",
+      "default": "./dist/metrics.js"
+    },
     "./publisher": {
       "types": "./dist/publisher/index.d.ts",
       "import": "./dist/publisher/index.js",
       "default": "./dist/publisher/index.js"
     },
+    "./runtime-config": {
+      "types": "./dist/runtime-config/index.d.ts",
+      "import": "./dist/runtime-config/index.js",
+      "default": "./dist/runtime-config/index.js"
+    },
     "./state-store": {
       "types": "./dist/state-store/index.d.ts",
       "import": "./dist/state-store/index.js",
       "default": "./dist/state-store/index.js"
     },
+    "./status": {
+      "types": "./dist/status/index.d.ts",
+      "import": "./dist/status/index.js",
+      "default": "./dist/status/index.js"
+    },
     "./webhook": {
       "types": "./dist/webhook/index.d.ts",
       "import": "./dist/webhook/index.js",
@@ -54,18 +80,21 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@effect/platform-node": "4.0.0-beta.83",
-    "@useairfoil/flight": "0.4.3",
-    "@useairfoil/wings": "0.4.1"
+    "@effect/platform-node": "^4.0.0-rc.108",
+    "@useairfoil/flight": "0.4.4",
+    "@useairfoil/wings": "0.4.2"
   },
   "devDependencies": {
-    "@effect/vitest": "4.0.0-beta.83",
+    "@effect/sql-pg": "^4.0.0-rc.108",
+    "@effect/vitest": "^4.0.0-rc.108",
     "@types/node": "^24.2.0",
+    "effect": "^4.0.0-rc.108",
+    "testcontainers": "^11.14.0",
     "tsdown": "^0.15.6",
-    "vitest": "^3.2.4"
+    "vitest": "^4.1.0"
   },
   "peerDependencies": {
-    "effect": "4.0.0-beta.83"
+    "effect": "^4.0.0-rc.108"
   },
   "peerDependenciesMeta": {
     "effect": {

package/src/auth/index.ts

@@ -0,0 +1 @@
+export * from "./token-cache";

package/src/auth/token-cache.ts

@@ -0,0 +1,54 @@
+import { Cache, Duration, Effect, Exit, Number, Redacted } from "effect";
+
+export interface ExpiringToken {
+  /** The credential kept in memory by the cache. */
+  readonly value: Redacted.Redacted<string>;
+  /** The lifetime reported by the provider. */
+  readonly expiresIn: Duration.Duration;
+}
+
+export interface TokenCache<E> {
+  /** Returns the cached token or performs one shared acquisition. */
+  readonly get: Effect.Effect<Redacted.Redacted<string>, E>;
+  /** Removes the cached token so the next `get` reacquires it. */
+  readonly invalidate: Effect.Effect<void>;
+}
+
+/**
+ * Creates a lazy, in-memory token cache around an acquisition effect.
+ *
+ * `refreshFactor` controls how much of the provider lifetime is cached. It is
+ * clamped between `0` and `1`.
+ *
+ * @example
+ * ```ts
+ * const cache = yield* makeTokenCache({
+ *   acquire: exchangeCredentials,
+ *   refreshFactor: 0.5,
+ * })
+ *
+ * const token = yield* cache.get
+ * ```
+ */
+export const makeTokenCache = <E, R>(options: {
+  readonly acquire: Effect.Effect<ExpiringToken, E, R>;
+  readonly refreshFactor?: number;
+}): Effect.Effect<TokenCache<E>, never, R> =>
+  Effect.gen(function* () {
+    const refreshFactor = Number.clamp(options.refreshFactor ?? 0.5, {
+      minimum: 0,
+      maximum: 1,
+    });
+    const cache = yield* Cache.makeWith((_: void) => options.acquire, {
+      capacity: 1,
+      timeToLive: (exit) =>
+        Exit.isSuccess(exit)
+          ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor)
+          : Duration.zero,
+    });
+
+    return {
+      get: Cache.get(cache, undefined).pipe(Effect.map((token) => token.value)),
+      invalidate: Cache.invalidate(cache, undefined),
+    };
+  });

package/src/connector-app/check.ts

@@ -0,0 +1,76 @@
+import { Context, Effect, Layer, Tuple } from "effect";
+
+import type { ConnectorDefinition, ResourceDefinition, ResourceName } from "../core/types";
+
+export type ResourceCheckResult =
+  | { readonly _tag: "ok" }
+  | { readonly _tag: "error"; readonly message: string };
+
+export type CheckOptions<Name extends string> = {
+  readonly resources: ReadonlyArray<Name>;
+};
+
+type ResourceCheckResults<Name extends PropertyKey> = {
+  readonly [Key in Name]: ResourceCheckResult;
+};
+
+export type CheckResult<
+  Resources extends ReadonlyArray<ResourceDefinition>,
+  Name extends ResourceName<Resources> = ResourceName<Resources>,
+> = ResourceCheckResults<Name>;
+
+type CheckFailure = { readonly message: string };
+
+const ok: ResourceCheckResult = { _tag: "ok" };
+
+const error = (message: string): ResourceCheckResult => ({ _tag: "error", message });
+
+const fromEntries = <Name extends string>(
+  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,
+): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;
+
+const resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>
+  fromEntries(names.map((name) => Tuple.make(name, result)));
+
+/**
+ * Builds the connector service from its layer and validates only the selected
+ * resources. The layer remains open until every check finishes.
+ */
+export function check<
+  Identifier,
+  const Resources extends ReadonlyArray<ResourceDefinition>,
+  E extends CheckFailure,
+  R,
+  const Name extends ResourceName<Resources>,
+>(
+  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,
+  layer: Layer.Layer<Identifier, E, R>,
+  options: CheckOptions<Name>,
+): Effect.Effect<CheckResult<Resources, Name>, never, R> {
+  if (options.resources.length === 0) {
+    return Effect.succeed(fromEntries<Name>([]));
+  }
+
+  return service.pipe(
+    Effect.flatMap((connector) =>
+      Effect.forEach(options.resources, (name) => {
+        const resource = connector.resources.find((candidate) => candidate.name === name);
+        const result = resource
+          ? resource.check.pipe(
+              Effect.match({
+                onFailure: (failure) => error(failure.message),
+                onSuccess: () => ok,
+              }),
+            )
+          : Effect.succeed(error(`Unknown connector resource: ${name}`));
+
+        return result.pipe(Effect.map((result) => Tuple.make(name, result)));
+      }).pipe(Effect.map(fromEntries)),
+    ),
+    Effect.provide(layer),
+    Effect.match({
+      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
+      onSuccess: (results) => results,
+    }),
+  );
+}

package/src/connector-app/index.ts

@@ -1 +1,2 @@
+export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from "./check";
 export { start, type App, type StartOptions } from "./start";

package/src/connector-app/start.ts

@@ -14,6 +14,8 @@ export type StartOptions = {
   readonly port: number;
   readonly initialCutoff?: Cursor.Value;
   readonly healthPath?: HttpRouter.PathInput;
+  readonly metricsPath?: HttpRouter.PathInput | false;
+  readonly statusPath?: HttpRouter.PathInput | false;
   readonly disableHttpLogger?: boolean;
 };
 
@@ -32,6 +34,8 @@ export const start = (app: App, options: StartOptions) =>
       webhook: {
         routes: app.webhooks ?? [],
         healthPath: options.healthPath ?? "/health",
+        metricsPath: options.metricsPath,
+        statusPath: options.statusPath,
         disableHttpLogger: options.disableHttpLogger ?? true,
       },
     }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));

package/src/core/builder.ts

@@ -21,9 +21,14 @@ export const Connector = {
 };
 
 export const Resource = {
-  entity: <S extends ResourceSchema, Payload = never, R = never>(
-    definition: ResourceDefinition<S, Payload, R>,
-  ): ResourceDefinition<S, Payload, R> => definition,
+  entity: <
+    S extends ResourceSchema,
+    Payload = never,
+    R = never,
+    const Name extends string = string,
+  >(
+    definition: ResourceDefinition<S, Payload, R, Name>,
+  ): ResourceDefinition<S, Payload, R, Name> => definition,
 
   webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>
     definition,

package/src/core/types.ts

@@ -37,12 +37,23 @@ export type ResourceState = {
     readonly cutoff: Cursor.Value;
     readonly pageCursor?: Cursor.Value;
     readonly completed: boolean;
+    readonly lastSuccessAt?: string;
   };
   readonly changes?: {
     readonly cursor: Cursor.Value;
+    readonly lastSuccessAt?: string;
+  };
+  readonly lastError?: {
+    readonly source: "backfill" | "changes";
+    readonly operation: "fetch" | "publish" | "checkpoint";
+    readonly code: "fetch_failed" | "publish_failed" | "checkpoint_failed";
+    readonly message: string;
+    readonly at: string;
   };
 };
 
+export type SyncState = "pending" | "backfilling" | "live" | "error";
+
 export type ResourceSchema = Schema.Decoder<object>;
 
 /**
@@ -112,14 +123,17 @@ export type ResourceDefinition<
   S extends ResourceSchema = ResourceSchema,
   Payload = unknown,
   R = never,
+  Name extends string = string,
 > = {
-  readonly name: string;
+  readonly name: Name;
   readonly schema: S;
   readonly key: ResourceField<NoInfer<S>>;
   readonly version: ResourceField<NoInfer<S>>;
   readonly partition?: {
     readonly required: boolean;
   };
+  /** Read-only validation used before provisioning this resource. */
+  readonly check: Effect.Effect<void, ConnectorError>;
   readonly backfill?: PageFetch<ResourceRow<NoInfer<S>>, R>;
   readonly changes?: ChangesFetch<ResourceRow<NoInfer<S>>, R>;
   readonly webhook?: WebhookHandler<ResourceRow<NoInfer<S>>, Payload>;
@@ -153,6 +167,10 @@ export type ResourceRows<Resources extends ReadonlyArray<ResourceDefinition>> =
     : never;
 };
 
+/** Resource names preserved from a connector's resource tuple. */
+export type ResourceName<Resources extends ReadonlyArray<ResourceDefinition>> =
+  keyof ResourceRows<Resources> & string;
+
 /**
  * Takes a resource definition and returns the payload type accepted by
  * `to(resource, payload)`.
@@ -192,7 +210,7 @@ export type WebhookRoute<
   readonly schema: Schema.Decoder<Payload>;
   handler(
     context: WebhookRouteContext<Resources, Payload>,
-  ): Effect.Effect<HttpServerResponse.HttpServerResponse, unknown>;
+  ): Effect.Effect<HttpServerResponse.HttpServerResponse, ConnectorError>;
 };
 
 export type WebhookRouteInput<

package/src/index.ts

@@ -1,8 +1,13 @@
 export * from "./core";
+export * as Auth from "./auth";
 export * as ConnectorApp from "./connector-app";
 export * as Ingestion from "./ingestion";
+export * as Manifest from "./manifest";
+export * as Metrics from "./metrics";
 export * as Publisher from "./publisher";
+export * as RuntimeConfig from "./runtime-config";
 export * as StateStore from "./state-store";
+export * as Status from "./status";
 export * as Telemetry from "./telemetry";
 export * as Webhook from "./webhook";
 export * from "./errors";