@useairfoil/connector-kit 0.6.0 → 0.6.1

package/src/catalog/table.ts

@@ -0,0 +1,201 @@
+import type {
+  IcebergType,
+  StructField,
+  TableIdentifier,
+  TableMetadata,
+  TableSchema,
+} from "iceberg-js";
+
+import * as Wings from "@useairfoil/wings";
+import { Effect, type Schema } from "effect";
+import { HttpClient } from "effect/http";
+import { getCurrentSchema } from "iceberg-js";
+
+import { ConnectorError } from "../errors";
+import { IcebergSchemaError } from "../iceberg/error";
+import { compileTable, makeCommitRequest } from "../iceberg/table";
+
+type Catalog = Effect.Success<
+  ReturnType<Wings.CatalogManager.CatalogManagerService["getIcebergCatalog"]>
+>;
+
+type TableOptions = {
+  readonly catalog: Catalog;
+  readonly identifier: TableIdentifier;
+  readonly location?: string | undefined;
+};
+
+type WingsTableOptions = {
+  readonly wingsUri: string;
+  readonly catalogId: string;
+  readonly identifier: TableIdentifier;
+  readonly location?: string | undefined;
+};
+
+const tableError = (cause: unknown): ConnectorError => {
+  if (cause instanceof ConnectorError) return cause;
+
+  const message = cause instanceof Error ? cause.message : "Failed to prepare Iceberg table";
+  return new ConnectorError({
+    message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,
+    cause,
+  });
+};
+
+const compareTypes = (
+  expected: IcebergType,
+  actual: IcebergType,
+  path: string,
+): string | undefined => {
+  if (typeof expected === "string" || typeof actual === "string") {
+    return expected === actual ? undefined : path;
+  }
+
+  if (expected.type !== actual.type) return path;
+
+  switch (expected.type) {
+    case "struct":
+      if (actual.type !== "struct") return path;
+      return compareFields(expected.fields, actual.fields, path);
+    case "list":
+      if (
+        actual.type !== "list" ||
+        expected["element-id"] !== actual["element-id"] ||
+        expected["element-required"] !== actual["element-required"]
+      ) {
+        return path;
+      }
+      return compareTypes(expected.element, actual.element, `${path}.element`);
+    case "map":
+      if (
+        actual.type !== "map" ||
+        expected["key-id"] !== actual["key-id"] ||
+        expected["value-id"] !== actual["value-id"] ||
+        expected["value-required"] !== actual["value-required"]
+      ) {
+        return path;
+      }
+      return (
+        compareTypes(expected.key, actual.key, `${path}.key`) ??
+        compareTypes(expected.value, actual.value, `${path}.value`)
+      );
+  }
+};
+
+const compareFields = (
+  expected: ReadonlyArray<StructField>,
+  actual: ReadonlyArray<StructField>,
+  path: string,
+): string | undefined => {
+  if (expected.length !== actual.length) return path;
+
+  for (const field of expected) {
+    const fieldPath = `${path}.${field.name}`;
+    const other = actual.find((candidate) => candidate.name === field.name);
+
+    if (!other || field.id !== other.id || field.required !== other.required) return fieldPath;
+
+    const mismatch = compareTypes(field.type, other.type, fieldPath);
+    if (mismatch) return mismatch;
+  }
+};
+
+const verifyTable = (
+  metadata: TableMetadata,
+  expected: TableSchema,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.try({
+    try: () => {
+      if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+      const actual = getCurrentSchema(metadata);
+      if (!actual) throw new Error("Table has no current schema");
+
+      const path = compareFields(expected.fields, actual.fields, "$");
+      if (path) {
+        throw new Error(
+          `Table schema does not match at ${path}. Migrate the table or revert the schema change.`,
+        );
+      }
+
+      const expectedIds = new Set(expected["identifier-field-ids"]);
+      const actualIds = new Set(actual["identifier-field-ids"]);
+      if (
+        expectedIds.size !== actualIds.size ||
+        [...expectedIds].some((id) => !actualIds.has(id))
+      ) {
+        throw new Error(
+          "Table identifier fields do not match. Recreate the table or revert the schema change.",
+        );
+      }
+
+      return actual;
+    },
+    catch: (cause) =>
+      new ConnectorError({
+        message: cause instanceof Error ? cause.message : "Invalid table metadata",
+        cause,
+      }),
+  });
+
+/** Creates a table with the IDs from its Effect schema, then checks the loaded schema. */
+export const createTable = (
+  schema: Schema.Top,
+  options: TableOptions,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.gen(function* () {
+    const table = yield* compileTable(schema);
+    yield* options.catalog.commitTable(
+      options.identifier,
+      makeCommitRequest(table, { location: options.location }),
+    );
+
+    const metadata = yield* options.catalog.loadTable(options.identifier);
+    return yield* verifyTable(metadata, table.schema);
+  }).pipe(Effect.mapError(tableError));
+
+/** Creates a table through a Wings catalog. */
+export const createTableFromWings = (
+  schema: Schema.Top,
+  options: WingsTableOptions,
+): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const manager = yield* Wings.CatalogManager.make({ baseUrl: options.wingsUri });
+      const catalog = yield* manager.getIcebergCatalog(options.catalogId);
+      return yield* createTable(schema, {
+        catalog,
+        identifier: options.identifier,
+        location: options.location,
+      });
+    }),
+  ).pipe(Effect.mapError(tableError));
+
+/** Loads a table, creating it when it is missing. */
+export const ensureTable = (
+  schema: Schema.Top,
+  options: TableOptions,
+): Effect.Effect<TableSchema, ConnectorError> =>
+  Effect.gen(function* () {
+    const table = yield* compileTable(schema);
+    const metadata = yield* options.catalog.loadTable(options.identifier).pipe(
+      Effect.catchIf(
+        (error) => error.status === 404,
+        () =>
+          options.catalog
+            .commitTable(
+              options.identifier,
+              makeCommitRequest(table, { location: options.location }),
+            )
+            .pipe(
+              // Another instance may have created the table. Reload and check it.
+              Effect.catchIf(
+                (error) => error.status === 409 || error.status === 412,
+                () => Effect.void,
+              ),
+              Effect.flatMap(() => options.catalog.loadTable(options.identifier)),
+            ),
+      ),
+    );
+
+    return yield* verifyTable(metadata, table.schema);
+  }).pipe(Effect.mapError(tableError));

package/src/connector-app/create-table.ts

@@ -0,0 +1,47 @@
+import { Effect, Option, type Schema } from "effect";
+import { Argument, CliError, Command, Flag } from "effect/cli";
+
+import { createTableFromWings } from "../catalog/table";
+import { wingsUri } from "../runtime-config/config";
+
+/** Creates the table command for a connector's row schemas. */
+export const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>
+  Command.make(
+    "create-table",
+    {
+      table: Argument.String("table"),
+      catalog: Flag.String("catalog"),
+      location: Flag.String("location").pipe(Flag.optional),
+      resource: Flag.String("resource").pipe(Flag.optional),
+    },
+    ({ table, catalog, location, resource }) =>
+      Effect.gen(function* () {
+        const namespace = table.split(".");
+        const name = namespace.pop();
+
+        if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) {
+          return yield* new CliError.UserError({
+            cause: "Invalid table name",
+            userMessage: "Use a table name like my.namespace.table",
+          });
+        }
+
+        const resourceName = Option.getOrElse(resource, () => name);
+        const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : undefined;
+        if (!schema) {
+          return yield* new CliError.UserError({
+            cause: "Unknown resource",
+            userMessage: `Unknown resource: ${resourceName}`,
+          });
+        }
+
+        const uri = yield* wingsUri;
+        yield* createTableFromWings(schema, {
+          wingsUri: uri.href,
+          catalogId: catalog,
+          identifier: { namespace, name },
+          location: Option.getOrUndefined(location),
+        });
+        yield* Effect.log(`Created ${table}`);
+      }),
+  ).pipe(Command.withDescription("Create an Iceberg table for a connector resource"));

package/src/connector-app/demo.ts

@@ -0,0 +1,42 @@
+import { ByteSize, DateTime, Effect, Option } from "effect";
+import { Command, Flag } from "effect/cli";
+
+import { run, type DemoDefinition } from "../demo";
+import { layerWingsConfig } from "../ingestor/wings";
+
+/** Reads sizes such as `512KiB`, `128MiB` or `1GiB`. */
+export const parseSize = (input: string): Option.Option<number> => {
+  if (!/^\d+(KiB|MiB|GiB)$/.test(input)) return Option.none();
+
+  return ByteSize.fromString(input).pipe(
+    Option.flatMap(ByteSize.toNumber),
+    Option.filter((bytes) => bytes > 0),
+  );
+};
+
+/**
+ * Creates the command that writes the demo data to Wings, then exits.
+ * It reads Wings and table bindings from Config, so provide `RuntimeConfig.layerHosted()`.
+ */
+export const makeDemoCommand = (demo: DemoDefinition) =>
+  Command.make(
+    "demo",
+    {
+      size: Flag.String("size").pipe(
+        Flag.filterMap(parseSize, () => "a size like 512KiB, 128MiB or 1GiB"),
+      ),
+      seed: Flag.Int("seed").pipe(Flag.withDefault(1)),
+    },
+    ({ size, seed }) =>
+      Effect.gen(function* () {
+        const now = yield* DateTime.nowAsDate;
+
+        yield* run(demo, { bytes: size, seed, now }).pipe(
+          Effect.provide(
+            layerWingsConfig({
+              resources: [...demo.setup.tables, ...demo.group.tables],
+            }),
+          ),
+        );
+      }),
+  ).pipe(Command.withDescription("Write demo data for the bound tables, then exit"));

package/src/connector-app/index.ts

@@ -1,2 +1,4 @@
 export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from "./check";
+export { makeCreateTableCommand } from "./create-table";
+export { makeDemoCommand, parseSize } from "./demo";
 export { start, type App, type StartOptions } from "./start";

package/src/connector-app/start.ts

@@ -1,6 +1,6 @@
 import { NodeHttpServer } from "@effect/platform-node";
 import { DateTime, Effect } from "effect";
-import { HttpRouter } from "effect/unstable/http";
+import { HttpRouter } from "effect/http";
 import { createServer } from "node:http";
 
 import type { ConnectorDefinition } from "../core";

package/src/core/builder.ts

@@ -1,22 +1,24 @@
-import { DateTime, Effect } from "effect";
+import { DateTime, Effect, Option } from "effect";
 
 import type {
+  ChangesFeed,
   ChangesFetch,
   ConnectorDefinition,
   Cursor as CursorType,
-  DeleteValue,
   PageFetch,
   ResourceDefinition,
-  ResourceMutation,
+  ResourceRequiredField,
   ResourceSchema,
-  WebhookHandler,
 } from "./types";
 
 import { ConnectorError } from "../errors";
 
 export const Connector = {
-  define: <const Resources extends ReadonlyArray<ResourceDefinition>>(
-    definition: ConnectorDefinition<Resources>,
+  define: <
+    const Resources extends ReadonlyArray<ResourceDefinition>,
+    const FeedResources extends ReadonlyArray<Resources[number]> = Resources,
+  >(
+    definition: ConnectorDefinition<Resources, FeedResources>,
   ) => definition,
 };
 
@@ -26,28 +28,18 @@ export const Resource = {
     Payload = never,
     R = never,
     const Name extends string = string,
+    const Version extends ResourceRequiredField<S> = ResourceRequiredField<S>,
   >(
-    definition: ResourceDefinition<S, Payload, R, Name>,
-  ): ResourceDefinition<S, Payload, R, Name> => definition,
-
-  webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>
-    definition,
-
-  upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: "upsert", row }),
-
-  delete: (options: {
-    readonly key: DeleteValue;
-    readonly version: DeleteValue;
-  }): ResourceMutation<never> => ({
-    op: "delete",
-    key: options.key,
-    version: options.version,
-  }),
+    definition: ResourceDefinition<S, Payload, R, Name, Version>,
+  ): ResourceDefinition<S, Payload, R, Name, Version> => definition,
 };
 
 export const Fetch = {
   page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,
   changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,
+  feed: <const Resources extends ReadonlyArray<ResourceDefinition>, R = never>(
+    definition: ChangesFeed<Resources, R>,
+  ) => definition,
 };
 
 const decodeFailure = (kind: string, value: unknown) =>
@@ -73,7 +65,7 @@ export const Cursor = {
   isoDateTime: (): CursorType.Definition<string> => ({
     kind: "isoDateTime",
     decode: (value) => {
-      if (typeof value !== "string" || Number.isNaN(Date.parse(value))) {
+      if (typeof value !== "string" || Option.isNone(DateTime.make(value))) {
         return Effect.fail(decodeFailure("ISO datetime", value));
       }
       return Effect.succeed(value);

package/src/core/index.ts

@@ -1,20 +1,30 @@
 export { Connector, Cursor, Fetch, Resource } from "./builder";
+export type { Identified } from "../iceberg/annotations";
 export type {
+  ChangesFeed,
   ChangesFetch,
   ConnectorDefinition,
   Cursor as CursorTypes,
+  FeedRows,
   FetchChangesResult,
+  FetchFeedResult,
   FetchPageResult,
   PageFetch,
   ResourceBatch,
   ResourceDefinition,
+  ResourceDelete,
   ResourceField,
-  ResourceMutation,
+  ResourceKey,
   ResourcePayload,
+  ResourceRequiredField,
   ResourceRow,
   ResourceRows,
   ResourceSchema,
   ResourceState,
+  ResourceUpdate,
+  ResourceUpdateOf,
+  TableResource,
+  TableSource,
   WebhookAckMode,
   WebhookHandler,
   WebhookRoute,

package/src/core/types.ts

@@ -1,7 +1,8 @@
 import type { Duration, Effect, Schema } from "effect";
-import type { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import type { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
 
 import type { ConnectorError } from "../errors";
+import type { Identified, RequiredField } from "../iceberg/annotations";
 
 export namespace Cursor {
   export type Value = string | number | Date;
@@ -14,22 +15,10 @@ export namespace Cursor {
   };
 }
 
-export type DeleteValue = string | number | boolean | Date;
-
-export type ResourceMutation<Row extends object = object> =
-  | {
-      readonly op: "upsert";
-      readonly row: Row;
-    }
-  | {
-      readonly op: "delete";
-      readonly key: DeleteValue;
-      readonly version: DeleteValue;
-    };
-
+/** Rows and an optional cursor from one resource. */
 export type ResourceBatch<Row extends object = object> = {
   readonly cursor?: Cursor.Value;
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
+  readonly rows: ReadonlyArray<Row>;
 };
 
 export type ResourceState = {
@@ -45,8 +34,8 @@ export type ResourceState = {
   };
   readonly lastError?: {
     readonly source: "backfill" | "changes";
-    readonly operation: "fetch" | "publish" | "checkpoint";
-    readonly code: "fetch_failed" | "publish_failed" | "checkpoint_failed";
+    readonly operation: "fetch" | "ingest" | "checkpoint";
+    readonly code: "fetch_failed" | "ingest_failed" | "checkpoint_failed";
     readonly message: string;
     readonly at: string;
   };
@@ -54,14 +43,17 @@ export type ResourceState = {
 
 export type SyncState = "pending" | "backfilling" | "live" | "error";
 
-export type ResourceSchema = Schema.Decoder<object>;
+/** A row schema with its key, set by `Iceberg.identifier`. */
+export type ResourceSchema = Schema.Decoder<object> & Identified<string>;
 
 /**
  * Takes a resource schema and returns its decoded row object type.
  *
  * @example
  * ```ts
- * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
+ * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String }).pipe(
+ *   Iceberg.identifier("id"),
+ * );
  * type Product = ResourceRow<typeof ProductSchema>;
  * // Result:
  * // type Product = { readonly id: string; readonly updatedAt: string }
@@ -74,18 +66,43 @@ export type ResourceRow<S extends ResourceSchema> = Schema.Schema.Type<S>;
  *
  * @example
  * ```ts
- * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
+ * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String }).pipe(
+ *   Iceberg.identifier("id"),
+ * );
  * type ProductField = ResourceField<typeof ProductSchema>;
  * // Result:
  * // type ProductField = "id" | "updatedAt"
  *
- * const key: ResourceField<typeof ProductSchema> = "id";
+ * const version: ResourceField<typeof ProductSchema> = "updatedAt";
  * ```
  */
-export type ResourceField<S extends ResourceSchema> = keyof ResourceRow<S> & string;
+type KnownResourceField<S extends ResourceSchema> = keyof ResourceRow<S> & string;
+
+export type ResourceField<S extends ResourceSchema> = [KnownResourceField<S>] extends [never]
+  ? string
+  : KnownResourceField<S>;
+
+/** A row field that cannot be null or undefined. */
+export type ResourceRequiredField<S extends ResourceSchema> = RequiredField<S>;
+
+/** The fields that identify a row, from `Iceberg.identifier`. */
+export type ResourceKey<S extends ResourceSchema> = S extends Identified<infer Key> ? Key : never;
+
+/** Deletes a row: its key, version, and `_deleted: true`. */
+export type ResourceDelete<S extends ResourceSchema, Version extends ResourceField<S>> = Pick<
+  ResourceRow<S>,
+  Extract<ResourceKey<S> | Version, keyof ResourceRow<S>>
+> & {
+  readonly _deleted: true;
+};
+
+/** A full row, or a delete. Wings replaces the whole row. */
+export type ResourceUpdate<S extends ResourceSchema, Version extends ResourceField<S>> =
+  | (ResourceRow<S> & { readonly _deleted?: boolean })
+  | ResourceDelete<S, Version>;
 
 export type FetchPageResult<Row extends object> = {
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
+  readonly rows: ReadonlyArray<Row>;
   readonly nextPageCursor?: Cursor.Value;
   readonly hasMore: boolean;
 };
@@ -100,8 +117,10 @@ export type PageFetch<Row extends object, R = never> = {
 };
 
 export type FetchChangesResult<Row extends object> = {
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
+  readonly rows: ReadonlyArray<Row>;
   readonly cursor: Cursor.Value;
+  /** When true, the engine runs the changes again without waiting for the interval. */
+  readonly hasMore?: boolean;
 };
 
 export type ChangesFetch<Row extends object, R = never> = {
@@ -113,10 +132,9 @@ export type ChangesFetch<Row extends object, R = never> = {
 };
 
 export type WebhookHandler<Row extends object, Payload> = {
+  /** Types the payload. The route decodes it before `to`. */
   readonly schema: Schema.Decoder<Payload>;
-  handler(input: {
-    readonly payload: Payload;
-  }): Effect.Effect<ReadonlyArray<ResourceMutation<Row>>, ConnectorError>;
+  handler(input: { readonly payload: Payload }): Effect.Effect<ReadonlyArray<Row>, ConnectorError>;
 };
 
 export type ResourceDefinition<
@@ -124,19 +142,28 @@ export type ResourceDefinition<
   Payload = unknown,
   R = never,
   Name extends string = string,
+  Version extends ResourceRequiredField<S> = ResourceRequiredField<S>,
 > = {
   readonly name: Name;
-  readonly schema: S;
-  readonly key: ResourceField<NoInfer<S>>;
-  readonly version: ResourceField<NoInfer<S>>;
-  readonly partition?: {
-    readonly required: boolean;
-  };
+  /** Types the rows, and names the key. The Iceberg table decides how they are encoded. */
+  readonly rowSchema: S;
+
+  /** Orders updates for the entity. */
+  readonly version: Version;
+
   /** Read-only validation used before provisioning this resource. */
   readonly check: Effect.Effect<void, ConnectorError>;
   readonly backfill?: PageFetch<ResourceRow<NoInfer<S>>, R>;
-  readonly changes?: ChangesFetch<ResourceRow<NoInfer<S>>, R>;
-  readonly webhook?: WebhookHandler<ResourceRow<NoInfer<S>>, Payload>;
+  readonly changes?: ChangesFetch<ResourceUpdate<NoInfer<S>, NoInfer<Version>>, R>;
+  readonly webhook?: WebhookHandler<ResourceUpdate<NoInfer<S>, NoInfer<Version>>, Payload>;
+};
+
+/** The parts of a resource its table needs. */
+export type TableResource = Pick<ResourceDefinition, "name" | "rowSchema" | "version">;
+
+/** Resources that write to tables, such as a connector or a demo. */
+export type TableSource = {
+  readonly resources: ReadonlyArray<TableResource>;
 };
 
 /**
@@ -161,7 +188,9 @@ export type ResourceRows<Resources extends ReadonlyArray<ResourceDefinition>> =
   readonly [Resource in Resources[number] as Resource["name"]]: Resource extends ResourceDefinition<
     infer S,
     unknown,
-    unknown
+    unknown,
+    string,
+    infer _Version
   >
     ? ResourceRow<S>
     : never;
@@ -183,9 +212,11 @@ export type ResourceName<Resources extends ReadonlyArray<ResourceDefinition>> =
  * ```
  */
 export type ResourcePayload<R> =
-  R extends ResourceDefinition<ResourceSchema, infer Payload, unknown> ? Payload : never;
+  R extends ResourceDefinition<ResourceSchema, infer Payload, unknown, string, string>
+    ? Payload
+    : never;
 
-export type WebhookAckMode = "after-enqueue" | "after-publish";
+export type WebhookAckMode = "after-enqueue" | "after-ingest";
 
 export type WebhookRouteContext<
   Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>,
@@ -194,7 +225,7 @@ export type WebhookRouteContext<
   readonly request: HttpServerRequest.HttpServerRequest;
   readonly rawBody: Uint8Array;
   readonly payload: Payload;
-  /** Decodes and collects mutations for a resource-owned webhook handler. */
+  /** Runs a resource's webhook handler on a decoded payload and collects its rows. */
   readonly to: <Resource extends Resources[number]>(
     resource: Resource,
     payload: ResourcePayload<Resource>,
@@ -218,11 +249,55 @@ export type WebhookRouteInput<
   Payload,
 > = WebhookRoute<Resources, Payload>;
 
+/**
+ * Takes a resource definition and returns the rows its changes and webhooks may
+ * return: full rows or deletes.
+ */
+export type ResourceUpdateOf<R> =
+  R extends ResourceDefinition<infer S, unknown, unknown, string, infer Version>
+    ? ResourceUpdate<S, Version>
+    : never;
+
+/** Rows from one feed run, by resource name. A missing resource has no rows. */
+export type FeedRows<Resources extends ReadonlyArray<ResourceDefinition>> = {
+  readonly [Resource in Resources[number] as Resource["name"]]?: ReadonlyArray<
+    ResourceUpdateOf<Resource>
+  >;
+};
+
+export type FetchFeedResult<Resources extends ReadonlyArray<ResourceDefinition>> = {
+  readonly rows: FeedRows<Resources>;
+  readonly cursor: Cursor.Value;
+  /** When true, the engine runs the feed again without waiting for the interval. */
+  readonly hasMore?: boolean;
+};
+
+/**
+ * One changes source for several resources, with one cursor. Use it when one
+ * provider log, such as an events API, covers them all.
+ */
+export type ChangesFeed<
+  Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>,
+  R = never,
+> = {
+  readonly resources: Resources;
+  readonly cursor: Cursor.Definition;
+  readonly interval?: Duration.Input;
+  readonly fetch: (input: {
+    readonly cursor: Cursor.Value;
+    /** The resources that run. Skip the work for the others. */
+    readonly selected: ReadonlySet<string>;
+  }) => Effect.Effect<FetchFeedResult<Resources>, ConnectorError, R>;
+};
+
 export type ConnectorDefinition<
   Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>,
+  FeedResources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>,
 > = {
   readonly name: string;
   readonly title?: string;
   readonly resources: Resources;
+  /** Shared changes for resources that do not define their own `changes`. */
+  readonly changes?: ChangesFeed<FeedResources>;
   readonly webhooks?: ReadonlyArray<WebhookRoute>;
 };