@useairfoil/connector-kit 0.4.0 → 0.5.1

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@useairfoil/connector-kit",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "repository": "useairfoil/typescript-sdk",
   "files": [
     "dist",
     "src",
+    "METRICS.md",
     "README.md"
   ],
   "type": "module",
@@ -15,6 +16,61 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
+    },
+    "./auth": {
+      "types": "./dist/auth/index.d.ts",
+      "import": "./dist/auth/index.js",
+      "default": "./dist/auth/index.js"
+    },
+    "./connector-app": {
+      "types": "./dist/connector-app/index.d.ts",
+      "import": "./dist/connector-app/index.js",
+      "default": "./dist/connector-app/index.js"
+    },
+    "./ingestion": {
+      "types": "./dist/ingestion/index.d.ts",
+      "import": "./dist/ingestion/index.js",
+      "default": "./dist/ingestion/index.js"
+    },
+    "./manifest": {
+      "types": "./dist/manifest/index.d.ts",
+      "import": "./dist/manifest/index.js",
+      "default": "./dist/manifest/index.js"
+    },
+    "./metrics": {
+      "types": "./dist/metrics.d.ts",
+      "import": "./dist/metrics.js",
+      "default": "./dist/metrics.js"
+    },
+    "./publisher": {
+      "types": "./dist/publisher/index.d.ts",
+      "import": "./dist/publisher/index.js",
+      "default": "./dist/publisher/index.js"
+    },
+    "./runtime-config": {
+      "types": "./dist/runtime-config/index.d.ts",
+      "import": "./dist/runtime-config/index.js",
+      "default": "./dist/runtime-config/index.js"
+    },
+    "./state-store": {
+      "types": "./dist/state-store/index.d.ts",
+      "import": "./dist/state-store/index.js",
+      "default": "./dist/state-store/index.js"
+    },
+    "./status": {
+      "types": "./dist/status/index.d.ts",
+      "import": "./dist/status/index.js",
+      "default": "./dist/status/index.js"
+    },
+    "./webhook": {
+      "types": "./dist/webhook/index.d.ts",
+      "import": "./dist/webhook/index.js",
+      "default": "./dist/webhook/index.js"
+    },
+    "./errors": {
+      "types": "./dist/errors/index.d.ts",
+      "import": "./dist/errors/index.js",
+      "default": "./dist/errors/index.js"
     }
   },
   "scripts": {
@@ -24,19 +80,21 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@useairfoil/flight": "0.4.0",
-    "@useairfoil/wings": "0.4.0",
-    "consola": "^3.4.2",
-    "json-stable-stringify": "^1.3.0"
+    "@effect/platform-node": "^4.0.0-rc.108",
+    "@useairfoil/flight": "0.4.4",
+    "@useairfoil/wings": "0.4.2"
   },
   "devDependencies": {
-    "@effect/vitest": "4.0.0-beta.54",
+    "@effect/sql-pg": "^4.0.0-rc.108",
+    "@effect/vitest": "^4.0.0-rc.108",
     "@types/node": "^24.2.0",
+    "effect": "^4.0.0-rc.108",
+    "testcontainers": "^11.14.0",
     "tsdown": "^0.15.6",
-    "vitest": "^3.2.4"
+    "vitest": "^4.1.0"
   },
   "peerDependencies": {
-    "effect": "4.0.0-beta.54"
+    "effect": "^4.0.0-rc.108"
   },
   "peerDependenciesMeta": {
     "effect": {

package/src/auth/index.ts

@@ -0,0 +1 @@
+export * from "./token-cache";

package/src/auth/token-cache.ts

@@ -0,0 +1,54 @@
+import { Cache, Duration, Effect, Exit, Number, Redacted } from "effect";
+
+export interface ExpiringToken {
+  /** The credential kept in memory by the cache. */
+  readonly value: Redacted.Redacted<string>;
+  /** The lifetime reported by the provider. */
+  readonly expiresIn: Duration.Duration;
+}
+
+export interface TokenCache<E> {
+  /** Returns the cached token or performs one shared acquisition. */
+  readonly get: Effect.Effect<Redacted.Redacted<string>, E>;
+  /** Removes the cached token so the next `get` reacquires it. */
+  readonly invalidate: Effect.Effect<void>;
+}
+
+/**
+ * Creates a lazy, in-memory token cache around an acquisition effect.
+ *
+ * `refreshFactor` controls how much of the provider lifetime is cached. It is
+ * clamped between `0` and `1`.
+ *
+ * @example
+ * ```ts
+ * const cache = yield* makeTokenCache({
+ *   acquire: exchangeCredentials,
+ *   refreshFactor: 0.5,
+ * })
+ *
+ * const token = yield* cache.get
+ * ```
+ */
+export const makeTokenCache = <E, R>(options: {
+  readonly acquire: Effect.Effect<ExpiringToken, E, R>;
+  readonly refreshFactor?: number;
+}): Effect.Effect<TokenCache<E>, never, R> =>
+  Effect.gen(function* () {
+    const refreshFactor = Number.clamp(options.refreshFactor ?? 0.5, {
+      minimum: 0,
+      maximum: 1,
+    });
+    const cache = yield* Cache.makeWith((_: void) => options.acquire, {
+      capacity: 1,
+      timeToLive: (exit) =>
+        Exit.isSuccess(exit)
+          ? Duration.millis(Duration.toMillis(exit.value.expiresIn) * refreshFactor)
+          : Duration.zero,
+    });
+
+    return {
+      get: Cache.get(cache, undefined).pipe(Effect.map((token) => token.value)),
+      invalidate: Cache.invalidate(cache, undefined),
+    };
+  });

package/src/connector-app/check.ts

@@ -0,0 +1,76 @@
+import { Context, Effect, Layer, Tuple } from "effect";
+
+import type { ConnectorDefinition, ResourceDefinition, ResourceName } from "../core/types";
+
+export type ResourceCheckResult =
+  | { readonly _tag: "ok" }
+  | { readonly _tag: "error"; readonly message: string };
+
+export type CheckOptions<Name extends string> = {
+  readonly resources: ReadonlyArray<Name>;
+};
+
+type ResourceCheckResults<Name extends PropertyKey> = {
+  readonly [Key in Name]: ResourceCheckResult;
+};
+
+export type CheckResult<
+  Resources extends ReadonlyArray<ResourceDefinition>,
+  Name extends ResourceName<Resources> = ResourceName<Resources>,
+> = ResourceCheckResults<Name>;
+
+type CheckFailure = { readonly message: string };
+
+const ok: ResourceCheckResult = { _tag: "ok" };
+
+const error = (message: string): ResourceCheckResult => ({ _tag: "error", message });
+
+const fromEntries = <Name extends string>(
+  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,
+): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;
+
+const resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>
+  fromEntries(names.map((name) => Tuple.make(name, result)));
+
+/**
+ * Builds the connector service from its layer and validates only the selected
+ * resources. The layer remains open until every check finishes.
+ */
+export function check<
+  Identifier,
+  const Resources extends ReadonlyArray<ResourceDefinition>,
+  E extends CheckFailure,
+  R,
+  const Name extends ResourceName<Resources>,
+>(
+  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,
+  layer: Layer.Layer<Identifier, E, R>,
+  options: CheckOptions<Name>,
+): Effect.Effect<CheckResult<Resources, Name>, never, R> {
+  if (options.resources.length === 0) {
+    return Effect.succeed(fromEntries<Name>([]));
+  }
+
+  return service.pipe(
+    Effect.flatMap((connector) =>
+      Effect.forEach(options.resources, (name) => {
+        const resource = connector.resources.find((candidate) => candidate.name === name);
+        const result = resource
+          ? resource.check.pipe(
+              Effect.match({
+                onFailure: (failure) => error(failure.message),
+                onSuccess: () => ok,
+              }),
+            )
+          : Effect.succeed(error(`Unknown connector resource: ${name}`));
+
+        return result.pipe(Effect.map((result) => Tuple.make(name, result)));
+      }).pipe(Effect.map(fromEntries)),
+    ),
+    Effect.provide(layer),
+    Effect.match({
+      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
+      onSuccess: (results) => results,
+    }),
+  );
+}

package/src/connector-app/index.ts

@@ -0,0 +1,2 @@
+export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from "./check";
+export { start, type App, type StartOptions } from "./start";

package/src/connector-app/start.ts

@@ -0,0 +1,42 @@
+import { NodeHttpServer } from "@effect/platform-node";
+import { DateTime, Effect } from "effect";
+import { HttpRouter } from "effect/unstable/http";
+import { createServer } from "node:http";
+
+import type { ConnectorDefinition } from "../core";
+import type { Cursor } from "../core/types";
+
+import { run as runIngestion } from "../ingestion";
+
+export type App = ConnectorDefinition;
+
+export type StartOptions = {
+  readonly port: number;
+  readonly initialCutoff?: Cursor.Value;
+  readonly healthPath?: HttpRouter.PathInput;
+  readonly metricsPath?: HttpRouter.PathInput | false;
+  readonly statusPath?: HttpRouter.PathInput | false;
+  readonly disableHttpLogger?: boolean;
+};
+
+export const start = (app: App, options: StartOptions) =>
+  Effect.gen(function* () {
+    const initialCutoff =
+      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+    const routes = app.webhooks?.map((route) => route.path) ?? [];
+
+    yield* Effect.logInfo("webhook server ready").pipe(
+      Effect.annotateLogs({ port: options.port, routes }),
+    );
+
+    return yield* runIngestion(app, {
+      initialCutoff,
+      webhook: {
+        routes: app.webhooks ?? [],
+        healthPath: options.healthPath ?? "/health",
+        metricsPath: options.metricsPath,
+        statusPath: options.statusPath,
+        disableHttpLogger: options.disableHttpLogger ?? true,
+      },
+    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
+  });

package/src/core/builder.ts

@@ -1,12 +1,84 @@
-import type { ConnectorDefinition, EntityDefinition, EntitySchema, EventDefinition } from "./types";
+import { DateTime, Effect } from "effect";
 
-export const defineConnector = <
-  const Entities extends ReadonlyArray<EntityDefinition<EntitySchema>>,
-  const Events extends ReadonlyArray<EventDefinition<EntitySchema>>,
->(
-  definition: ConnectorDefinition<Entities, Events>,
-) => definition;
+import type {
+  ChangesFetch,
+  ConnectorDefinition,
+  Cursor as CursorType,
+  DeleteValue,
+  PageFetch,
+  ResourceDefinition,
+  ResourceMutation,
+  ResourceSchema,
+  WebhookHandler,
+} from "./types";
 
-export const defineEntity = <S extends EntitySchema>(definition: EntityDefinition<S>) => definition;
+import { ConnectorError } from "../errors";
 
-export const defineEvent = <S extends EntitySchema>(definition: EventDefinition<S>) => definition;
+export const Connector = {
+  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(
+    definition: ConnectorDefinition<Resources>,
+  ) => definition,
+};
+
+export const Resource = {
+  entity: <
+    S extends ResourceSchema,
+    Payload = never,
+    R = never,
+    const Name extends string = string,
+  >(
+    definition: ResourceDefinition<S, Payload, R, Name>,
+  ): ResourceDefinition<S, Payload, R, Name> => definition,
+
+  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>
+    definition,
+
+  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: "upsert", row }),
+
+  delete: (options: {
+    readonly key: DeleteValue;
+    readonly version: DeleteValue;
+  }): ResourceMutation<never> => ({
+    op: "delete",
+    key: options.key,
+    version: options.version,
+  }),
+};
+
+export const Fetch = {
+  page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,
+  changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,
+};
+
+const decodeFailure = (kind: string, value: unknown) =>
+  new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
+
+export const Cursor = {
+  string: (): CursorType.Definition<string> => ({
+    kind: "string",
+    decode: (value) =>
+      typeof value === "string"
+        ? Effect.succeed(value)
+        : Effect.fail(decodeFailure("string", value)),
+  }),
+
+  number: (): CursorType.Definition<number> => ({
+    kind: "number",
+    decode: (value) =>
+      typeof value === "number" && Number.isFinite(value)
+        ? Effect.succeed(value)
+        : Effect.fail(decodeFailure("number", value)),
+  }),
+
+  isoDateTime: (): CursorType.Definition<string> => ({
+    kind: "isoDateTime",
+    decode: (value) => {
+      if (typeof value !== "string" || Number.isNaN(Date.parse(value))) {
+        return Effect.fail(decodeFailure("ISO datetime", value));
+      }
+      return Effect.succeed(value);
+    },
+  }),
+
+  nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),
+};

package/src/core/index.ts

@@ -0,0 +1,22 @@
+export { Connector, Cursor, Fetch, Resource } from "./builder";
+export type {
+  ChangesFetch,
+  ConnectorDefinition,
+  Cursor as CursorTypes,
+  FetchChangesResult,
+  FetchPageResult,
+  PageFetch,
+  ResourceBatch,
+  ResourceDefinition,
+  ResourceField,
+  ResourceMutation,
+  ResourcePayload,
+  ResourceRow,
+  ResourceRows,
+  ResourceSchema,
+  ResourceState,
+  WebhookAckMode,
+  WebhookHandler,
+  WebhookRoute,
+  WebhookRouteContext,
+} from "./types";