@useairfoil/connector-kit 0.5.3 → 0.6.1

package/dist/types-B35J5YPu.d.ts

@@ -1,174 +0,0 @@
-import { t as ConnectorError } from "./index-D54ySAKn.js";
-import { Duration, Effect, Schema } from "effect";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
-
-//#region src/core/types.d.ts
-declare namespace Cursor {
-  type Value = string | number | Date;
-  type Kind = "string" | "number" | "isoDateTime";
-  type Definition<T extends Value = Value> = {
-    readonly kind: Kind;
-    readonly decode: (value: unknown) => Effect.Effect<T, ConnectorError>;
-  };
-}
-type DeleteValue = string | number | boolean | Date;
-type ResourceMutation<Row extends object = object> = {
-  readonly op: "upsert";
-  readonly row: Row;
-} | {
-  readonly op: "delete";
-  readonly key: DeleteValue;
-  readonly version: DeleteValue;
-};
-type ResourceBatch<Row extends object = object> = {
-  readonly cursor?: Cursor.Value;
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
-};
-type ResourceState = {
-  readonly backfill?: {
-    readonly cutoff: Cursor.Value;
-    readonly pageCursor?: Cursor.Value;
-    readonly completed: boolean;
-    readonly lastSuccessAt?: string;
-  };
-  readonly changes?: {
-    readonly cursor: Cursor.Value;
-    readonly lastSuccessAt?: string;
-  };
-  readonly lastError?: {
-    readonly source: "backfill" | "changes";
-    readonly operation: "fetch" | "publish" | "checkpoint";
-    readonly code: "fetch_failed" | "publish_failed" | "checkpoint_failed";
-    readonly message: string;
-    readonly at: string;
-  };
-};
-type SyncState = "pending" | "backfilling" | "live" | "error";
-type ResourceSchema = Schema.Decoder<object>;
-/**
- * Takes a resource schema and returns its decoded row object type.
- *
- * @example
- * ```ts
- * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
- * type Product = ResourceRow<typeof ProductSchema>;
- * // Result:
- * // type Product = { readonly id: string; readonly updatedAt: string }
- * ```
- */
-type ResourceRow<S$1 extends ResourceSchema> = Schema.Schema.Type<S$1>;
-/**
- * Takes a resource schema and returns the string keys available on its row type.
- *
- * @example
- * ```ts
- * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
- * type ProductField = ResourceField<typeof ProductSchema>;
- * // Result:
- * // type ProductField = "id" | "updatedAt"
- *
- * const key: ResourceField<typeof ProductSchema> = "id";
- * ```
- */
-type ResourceField<S$1 extends ResourceSchema> = keyof ResourceRow<S$1> & string;
-type FetchPageResult<Row extends object> = {
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
-  readonly nextPageCursor?: Cursor.Value;
-  readonly hasMore: boolean;
-};
-type PageFetch<Row extends object, R = never> = {
-  readonly pageCursor: Cursor.Definition;
-  readonly cutoff: Cursor.Definition;
-  readonly fetch: (input: {
-    readonly pageCursor?: Cursor.Value;
-    readonly cutoff: Cursor.Value;
-  }) => Effect.Effect<FetchPageResult<Row>, ConnectorError, R>;
-};
-type FetchChangesResult<Row extends object> = {
-  readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
-  readonly cursor: Cursor.Value;
-};
-type ChangesFetch<Row extends object, R = never> = {
-  readonly cursor: Cursor.Definition;
-  readonly interval?: Duration.Input;
-  readonly fetch: (input: {
-    readonly cursor: Cursor.Value;
-  }) => Effect.Effect<FetchChangesResult<Row>, ConnectorError, R>;
-};
-type WebhookHandler<Row extends object, Payload$1> = {
-  readonly schema: Schema.Decoder<Payload$1>;
-  handler(input: {
-    readonly payload: Payload$1;
-  }): Effect.Effect<ReadonlyArray<ResourceMutation<Row>>, ConnectorError>;
-};
-type ResourceDefinition<S$1 extends ResourceSchema = ResourceSchema, Payload$1 = unknown, R = never, Name extends string = string> = {
-  readonly name: Name;
-  readonly schema: S$1;
-  readonly key: ResourceField<NoInfer<S$1>>;
-  readonly version: ResourceField<NoInfer<S$1>>;
-  readonly partition?: {
-    readonly required: boolean;
-  };
-  /** Read-only validation used before provisioning this resource. */
-  readonly check: Effect.Effect<void, ConnectorError>;
-  readonly backfill?: PageFetch<ResourceRow<NoInfer<S$1>>, R>;
-  readonly changes?: ChangesFetch<ResourceRow<NoInfer<S$1>>, R>;
-  readonly webhook?: WebhookHandler<ResourceRow<NoInfer<S$1>>, Payload$1>;
-};
-/**
- * Takes a connector resource tuple and returns a `{ [resourceName]: rowType }`
- * map.
- *
- * @example
- * ```ts
- * type Rows = ResourceRows<[typeof Products, typeof Orders]>;
- * // Result:
- * // type Rows = {
- * //   readonly products: Product;
- * //   readonly orders: Order;
- * // }
- *
- * type ProductRow = Rows["products"];
- * // Result:
- * // type ProductRow = Product
- * ```
- */
-type ResourceRows<Resources extends ReadonlyArray<ResourceDefinition>> = { readonly [Resource in Resources[number] as Resource["name"]]: Resource extends ResourceDefinition<infer S, unknown, unknown> ? ResourceRow<S> : never };
-/** Resource names preserved from a connector's resource tuple. */
-type ResourceName<Resources extends ReadonlyArray<ResourceDefinition>> = keyof ResourceRows<Resources> & string;
-/**
- * Takes a resource definition and returns the payload type accepted by
- * `to(resource, payload)`.
- *
- * @example
- * ```ts
- * type ProductPayload = ResourcePayload<typeof Products>;
- * // Result, if Products was defined with a ProductEvent webhook schema:
- * // type ProductPayload = ProductEvent
- * ```
- */
-type ResourcePayload<R> = R extends ResourceDefinition<ResourceSchema, infer Payload, unknown> ? Payload : never;
-type WebhookAckMode = "after-enqueue" | "after-publish";
-type WebhookRouteContext<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload$1 = unknown> = {
-  readonly request: HttpServerRequest.HttpServerRequest;
-  readonly rawBody: Uint8Array;
-  readonly payload: Payload$1;
-  /** Decodes and collects mutations for a resource-owned webhook handler. */
-  readonly to: <Resource$1 extends Resources[number]>(resource: Resource$1, payload: ResourcePayload<Resource$1>) => Effect.Effect<void, ConnectorError>;
-};
-type WebhookRoute<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload$1 = unknown> = {
-  readonly path: HttpRouter.PathInput;
-  readonly ackMode: WebhookAckMode;
-  readonly schema: Schema.Decoder<Payload$1>;
-  handler(context: WebhookRouteContext<Resources, Payload$1>): Effect.Effect<HttpServerResponse.HttpServerResponse, ConnectorError>;
-};
-type WebhookRouteInput<Resources extends ReadonlyArray<ResourceDefinition>, Payload$1> = WebhookRoute<Resources, Payload$1>;
-type ConnectorDefinition<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>> = {
-  readonly name: string;
-  readonly title?: string;
-  readonly resources: Resources;
-  readonly webhooks?: ReadonlyArray<WebhookRoute>;
-};
-//#endregion
-export { WebhookRouteInput as C, WebhookRouteContext as S, ResourceState as _, FetchChangesResult as a, WebhookHandler as b, ResourceBatch as c, ResourceMutation as d, ResourceName as f, ResourceSchema as g, ResourceRows as h, DeleteValue as i, ResourceDefinition as l, ResourceRow as m, ConnectorDefinition as n, FetchPageResult as o, ResourcePayload as p, Cursor as r, PageFetch as s, ChangesFetch as t, ResourceField as u, SyncState as v, WebhookRoute as x, WebhookAckMode as y };
-//# sourceMappingURL=types-B35J5YPu.d.ts.map

package/dist/webhook/index.d.ts

@@ -1,6 +0,0 @@
-import "../index-D54ySAKn.js";
-import { C as WebhookRouteInput, S as WebhookRouteContext, x as WebhookRoute, y as WebhookAckMode } from "../types-B35J5YPu.js";
-import "../service-IE5qaf9x.js";
-import "../index-Byr0Lf0C.js";
-import { n as route, r as router } from "../index-DKWIh6M1.js";
-export { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput, route, router };

package/dist/webhook/index.js

@@ -1,8 +0,0 @@
-import "../errors-DHWylqRZ.js";
-import "../metrics-CWPy2Aqc.js";
-import "../constants-BRhIX_rz.js";
-import "../service-DuFiQwid.js";
-import { n as router } from "../server-CXTQ7odl.js";
-import { t as route } from "../webhook-DI-Afxkg.js";
-
-export { route, router };

package/dist/webhook-DI-Afxkg.js

@@ -1,13 +0,0 @@
-import { t as __export } from "./chunk-Bp6m_JJh.js";
-import { n as router } from "./server-CXTQ7odl.js";
-
-//#region src/webhook/index.ts
-var webhook_exports = /* @__PURE__ */ __export({
-	route: () => route,
-	router: () => router
-});
-const route = (definition) => definition;
-
-//#endregion
-export { webhook_exports as n, route as t };
-//# sourceMappingURL=webhook-DI-Afxkg.js.map

package/dist/webhook-DI-Afxkg.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"webhook-DI-Afxkg.js","names":[],"sources":["../src/webhook/index.ts"],"sourcesContent":["import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from \"../core/types\";\n\nexport { router } from \"./server\";\nexport type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from \"./types\";\n\nexport const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(\n  definition: WebhookRouteInput<Resources, Payload>,\n): WebhookRoute<Resources, Payload> => definition;\n"],"mappings":";;;;;;;;AAKA,MAAa,SACX,eACqC"}

package/src/publisher/console.ts

@@ -1,18 +0,0 @@
-import { Effect, Layer } from "effect";
-
-import { Publisher } from "./service";
-
-export const layerConsole = Layer.succeed(Publisher)({
-  publish: ({ resource, source, batch }) =>
-    Effect.gen(function* () {
-      yield* Effect.logInfo(`[publisher] -> Source: ${source} | Resource: ${resource}`).pipe(
-        Effect.annotateLogs({
-          mutations: batch.mutations.length,
-          cursor: batch.cursor,
-          source,
-        }),
-      );
-
-      return { status: "accepted" as const, resource };
-    }),
-});

package/src/publisher/index.ts

@@ -1,7 +0,0 @@
-export type { ResourceBatch } from "../core/types";
-export type { PublishAck, PublishOptions, PublisherService, PublishSource } from "./service";
-export type { WingsPublisherConfig } from "./wings";
-
-export { layerConsole } from "./console";
-export { Publisher } from "./service";
-export { layerWings } from "./wings";

package/src/publisher/instrumented.ts

@@ -1,68 +0,0 @@
-import { Effect, Metric } from "effect";
-
-import type { ResourceBatch } from "../core/types";
-
-import { ConnectorError } from "../errors";
-import * as Metrics from "../metrics";
-import { Attr, EventAttr, EventName, addCurrentSpanEvent } from "../telemetry";
-import { Publisher, type PublishSource } from "./service";
-
-export const publishBatch = Effect.fnUntraced(function* (options: {
-  readonly connector?: string;
-  readonly resource: string;
-  readonly source: PublishSource;
-  readonly batch: ResourceBatch;
-}) {
-  const metric = {
-    connector: options.connector ?? "unknown",
-    resource: options.resource,
-    source: options.source,
-  };
-  const upserts = options.batch.mutations.filter((mutation) => mutation.op === "upsert").length;
-  const deletes = options.batch.mutations.length - upserts;
-
-  const publisher = yield* Publisher;
-  const [duration, ack] = yield* Effect.timed(
-    publisher.publish({
-      resource: options.resource,
-      source: options.source,
-      batch: options.batch,
-    }),
-  ).pipe(
-    Effect.catchCause((cause) =>
-      Metric.update(
-        Metric.withAttributes(Metrics.batchesTotal, { ...metric, outcome: "error" }),
-        1,
-      ).pipe(Effect.andThen(Effect.failCause(cause))),
-    ),
-  );
-
-  yield* Metric.update(Metric.withAttributes(Metrics.publishDuration, metric), duration);
-  yield* Metric.update(
-    Metric.withAttributes(Metrics.batchesTotal, { ...metric, outcome: ack.status }),
-    1,
-  );
-
-  yield* Effect.annotateCurrentSpan({ [Attr.publisherSuccess]: ack.status === "accepted" });
-  if (ack.status === "rejected") {
-    yield* Effect.annotateCurrentSpan({ [Attr.errorPhase]: "publish" });
-    return yield* Effect.fail(
-      new ConnectorError({
-        message: `Publisher rejected batch for ${options.resource}: ${ack.reason}`,
-      }),
-    );
-  }
-
-  yield* Metric.update(Metric.withAttributes(Metrics.entitiesUpsertedTotal, metric), upserts);
-  yield* Metric.update(Metric.withAttributes(Metrics.entitiesDeletedTotal, metric), deletes);
-  yield* Metric.update(
-    Metric.withAttributes(Metrics.batchSize, metric),
-    options.batch.mutations.length,
-  );
-
-  if (options.batch.cursor !== undefined) {
-    yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
-      [EventAttr.batchCursor]: options.batch.cursor,
-    });
-  }
-});

package/src/publisher/service.ts

@@ -1,37 +0,0 @@
-import type * as Wings from "@useairfoil/wings";
-import type * as Effect from "effect/Effect";
-
-import { Context } from "effect";
-
-import type { ResourceBatch } from "../core/types";
-import type { ConnectorError } from "../errors";
-
-export type PublishSource = "backfill" | "changes" | "webhook";
-
-export type PublishAck =
-  | {
-      readonly status: "accepted";
-      readonly resource: string;
-      readonly partition?: Wings.PartitionValue.PartitionValue;
-    }
-  | {
-      readonly status: "rejected";
-      readonly resource: string;
-      readonly reason: string;
-      readonly rejectedRows?: number;
-      readonly partition?: Wings.PartitionValue.PartitionValue;
-    };
-
-export type PublishOptions = {
-  readonly resource: string;
-  readonly source: PublishSource;
-  readonly batch: ResourceBatch;
-};
-
-export interface PublisherService {
-  readonly publish: (options: PublishOptions) => Effect.Effect<PublishAck, ConnectorError>;
-}
-
-export class Publisher extends Context.Service<Publisher, PublisherService>()(
-  "@useairfoil/connector-kit/Publisher",
-) {}

package/src/publisher/wings.ts

@@ -1,204 +0,0 @@
-import * as Wings from "@useairfoil/wings";
-import { Effect, Layer } from "effect";
-
-import type { ConnectorDefinition, ResourceDefinition } from "../core/types";
-
-import { ConnectorError } from "../errors";
-import { SpanName } from "../telemetry";
-import { Publisher, type PublishAck, type PublisherService } from "./service";
-
-type Rows = Record<string, unknown>;
-
-export type WingsTableMapping = {
-  readonly name: string;
-  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
-};
-
-export type WingsPublisherConfig = {
-  readonly connector: ConnectorDefinition;
-  readonly tables: Record<string, string | WingsTableMapping>;
-};
-
-type PublisherEntry = {
-  readonly resource: ResourceDefinition;
-  readonly table: Wings.Cluster.Table.Table;
-  readonly publisher: Wings.WingsClient.Publisher;
-  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
-  readonly keyField: Wings.Cluster.ArrowType.Field;
-  readonly versionField: Wings.Cluster.ArrowType.Field;
-};
-
-const normalizeMapping = (mapping: string | WingsTableMapping): WingsTableMapping =>
-  typeof mapping === "string" ? { name: mapping } : mapping;
-
-const buildRecordBatch = (rows: ReadonlyArray<Rows>) => {
-  const table = Wings.Arrow.tableFromJSON(Array.from(rows));
-  const [batch] = table.batches;
-  return batch
-    ? Effect.succeed(batch)
-    : Effect.fail(new ConnectorError({ message: "No rows to publish" }));
-};
-
-const validateTableMapping = (options: {
-  readonly resource: ResourceDefinition;
-  readonly table: Wings.Cluster.Table.Table;
-  readonly partitionValue?: Wings.PartitionValue.PartitionValue;
-}) =>
-  Effect.try({
-    try: () => {
-      const keyField = Wings.TableUtils.getKeyField(options.table);
-      const versionField = Wings.TableUtils.getVersionField(options.table);
-      const partitionField = Wings.TableUtils.getPartitionField(options.table);
-
-      if (keyField.name !== options.resource.key) {
-        throw new Error(
-          `Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`,
-        );
-      }
-      if (versionField.name !== options.resource.version) {
-        throw new Error(
-          `Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`,
-        );
-      }
-      if (options.resource.partition?.required === true && !options.partitionValue) {
-        throw new Error(`Resource ${options.resource.name} requires a partition value`);
-      }
-      if (partitionField && !options.partitionValue) {
-        throw new Error(
-          `Wings table ${options.table.name} is partitioned but no partition value was provided`,
-        );
-      }
-      if (!partitionField && options.partitionValue) {
-        throw new Error(
-          `Wings table ${options.table.name} is not partitioned but a partition value was provided`,
-        );
-      }
-
-      return { keyField, versionField };
-    },
-    catch: (cause) =>
-      new ConnectorError({
-        message: `Invalid Wings table mapping for ${options.resource.name}`,
-        cause,
-      }),
-  });
-
-const accepted = (entry: PublisherEntry): PublishAck => ({
-  status: "accepted",
-  resource: entry.resource.name,
-  partition: entry.partitionValue,
-});
-
-const rejected = (entry: PublisherEntry, reason: string, rejectedRows: number): PublishAck => ({
-  status: "rejected",
-  resource: entry.resource.name,
-  reason,
-  rejectedRows,
-  partition: entry.partitionValue,
-});
-
-export const layerWings = (
-  config: WingsPublisherConfig,
-): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>
-  Layer.effect(Publisher)(
-    Effect.gen(function* () {
-      const entries = new Map<string, PublisherEntry>();
-      const clusterClient = yield* Wings.WingsClient.clusterClient;
-
-      for (const resource of config.connector.resources) {
-        const mappingInput = config.tables[resource.name];
-        if (!mappingInput) {
-          return yield* Effect.fail(
-            new ConnectorError({ message: `Missing table for ${resource.name}` }),
-          );
-        }
-
-        const mapping = normalizeMapping(mappingInput);
-        const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(
-          Effect.mapError(
-            (error) =>
-              new ConnectorError({
-                message: `Failed to resolve Wings table for ${resource.name}`,
-                cause: error,
-              }),
-          ),
-        );
-        const { keyField, versionField } = yield* validateTableMapping({
-          resource,
-          table,
-          partitionValue: mapping.partitionValue,
-        });
-
-        const publisher = yield* Wings.WingsClient.publisher({
-          table,
-          partitionValue: mapping.partitionValue,
-        }).pipe(
-          Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),
-        );
-
-        entries.set(resource.name, {
-          resource,
-          table,
-          publisher,
-          partitionValue: mapping.partitionValue,
-          keyField,
-          versionField,
-        });
-      }
-
-      const pushRows = (
-        entry: PublisherEntry,
-        operation: "upsert" | "delete",
-        rows: ReadonlyArray<Rows>,
-      ) =>
-        Effect.gen(function* () {
-          if (rows.length === 0) return accepted(entry);
-          const batch = yield* buildRecordBatch(rows);
-          return yield* entry.publisher
-            .push({ operation, batch, partitionValue: entry.partitionValue })
-            .pipe(
-              Effect.matchEffect({
-                onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),
-                onSuccess: () => Effect.succeed(accepted(entry)),
-              }),
-            );
-        });
-
-      const service: PublisherService = {
-        publish: Effect.fn(SpanName.publish, { kind: "producer" })(function* ({
-          resource,
-          source: _source,
-          batch,
-        }) {
-          const entry = entries.get(resource);
-          if (!entry) {
-            return yield* Effect.fail(
-              new ConnectorError({ message: `Unknown resource ${resource}` }),
-            );
-          }
-
-          if (batch.mutations.length === 0) return accepted(entry);
-
-          const upserts: Rows[] = [];
-          const deletes: Rows[] = [];
-          for (const mutation of batch.mutations) {
-            if (mutation.op === "upsert") {
-              upserts.push(mutation.row as Rows);
-            } else {
-              deletes.push({
-                [entry.keyField.name]: mutation.key,
-                [entry.versionField.name]: mutation.version,
-              });
-            }
-          }
-
-          const upsertAck = yield* pushRows(entry, "upsert", upserts);
-          if (upsertAck.status === "rejected") return upsertAck;
-
-          return yield* pushRows(entry, "delete", deletes);
-        }),
-      };
-
-      return Publisher.of(service);
-    }),
-  );