@useairfoil/producer-shopify 0.0.1 → 0.2.0

package/src/auth.ts

@@ -0,0 +1,116 @@
+import { Auth } from "@useairfoil/connector-kit";
+import { Config, Context, Data, Duration, Effect, Layer, Redacted, Schema } from "effect";
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
+
+import type { ShopifyConfig } from "./manifest";
+
+type ShopifyAuthConfig = Pick<ShopifyConfig, "shopDomain" | "clientId" | "clientSecret">;
+
+export type ShopifyAuthErrorReason = "shop_not_permitted" | "invalid_client" | "exchange_failed";
+
+export class ShopifyAuthError extends Data.TaggedError("ShopifyAuthError")<{
+  readonly reason: ShopifyAuthErrorReason;
+  readonly message: string;
+}> {}
+
+export type ShopifyAuthService = Auth.TokenCache<ShopifyAuthError>;
+
+export class ShopifyAuth extends Context.Service<ShopifyAuth, ShopifyAuthService>()(
+  "@useairfoil/producer-shopify/ShopifyAuth",
+) {}
+
+const TokenResponseSchema = Schema.Struct({
+  access_token: Schema.String,
+  expires_in: Schema.Number,
+});
+
+const ErrorResponseSchema = Schema.Struct({
+  error: Schema.optional(Schema.String),
+});
+
+const normalizedShopDomain = (shopDomain: string): string =>
+  shopDomain.replace(/^https?:\/\//i, "").replace(/\/+$/g, "");
+
+const errorFor = (reason: ShopifyAuthErrorReason): ShopifyAuthError => {
+  switch (reason) {
+    case "shop_not_permitted":
+      return new ShopifyAuthError({
+        reason,
+        message: "The Shopify app and store must belong to the same Shopify organization",
+      });
+    case "invalid_client":
+      return new ShopifyAuthError({
+        reason,
+        message: "The Shopify client ID or client secret is invalid",
+      });
+    case "exchange_failed":
+      return new ShopifyAuthError({
+        reason,
+        message: "Shopify authentication failed",
+      });
+  }
+};
+
+const decodeRejectedResponse = (
+  response: HttpClientResponse.HttpClientResponse,
+): Effect.Effect<ShopifyAuthErrorReason> =>
+  HttpClientResponse.schemaBodyJson(ErrorResponseSchema)(response).pipe(
+    Effect.map((body) => {
+      switch (body.error) {
+        case "shop_not_permitted":
+          return "shop_not_permitted";
+        case "invalid_client":
+          return "invalid_client";
+        default:
+          return "exchange_failed";
+      }
+    }),
+    Effect.catch(() => Effect.succeed<ShopifyAuthErrorReason>("exchange_failed")),
+  );
+
+export const make = Effect.fnUntraced(function* (config: ShopifyAuthConfig) {
+  const client = yield* HttpClient.HttpClient;
+  const endpoint = `https://${normalizedShopDomain(config.shopDomain)}/admin/oauth/access_token`;
+
+  return yield* Auth.makeTokenCache({
+    acquire: Effect.scoped(
+      Effect.gen(function* () {
+        const request = HttpClientRequest.post(endpoint).pipe(
+          HttpClientRequest.bodyUrlParams({
+            grant_type: "client_credentials",
+            client_id: config.clientId,
+            client_secret: Redacted.value(config.clientSecret),
+          }),
+        );
+        const response = yield* client
+          .execute(request)
+          .pipe(Effect.mapError(() => errorFor("exchange_failed")));
+        const successfulResponse = yield* HttpClientResponse.filterStatusOk(response).pipe(
+          Effect.catchTag("HttpClientError", () =>
+            decodeRejectedResponse(response).pipe(
+              Effect.flatMap((reason) => Effect.fail(errorFor(reason))),
+            ),
+          ),
+        );
+        const body = yield* HttpClientResponse.schemaBodyJson(TokenResponseSchema)(
+          successfulResponse,
+        ).pipe(Effect.mapError(() => errorFor("exchange_failed")));
+
+        return {
+          value: Redacted.make(body.access_token),
+          expiresIn: Duration.seconds(body.expires_in),
+        };
+      }),
+    ),
+  });
+});
+
+export const layer = (
+  config: ShopifyAuthConfig,
+): Layer.Layer<ShopifyAuth, never, HttpClient.HttpClient> =>
+  Layer.effect(ShopifyAuth)(make(config));
+
+export const layerConfig = (
+  config: Config.Wrap<ShopifyAuthConfig>,
+): Layer.Layer<ShopifyAuth, Config.ConfigError, HttpClient.HttpClient> =>
+  Layer.unwrap(Config.unwrap(config).pipe(Effect.map(layer)));

package/src/connector.ts

@@ -0,0 +1,268 @@
+import {
+  Connector,
+  ConnectorError,
+  Cursor,
+  Fetch,
+  Resource,
+  Webhook,
+} from "@useairfoil/connector-kit";
+import { Config, Context, Effect, Layer, Redacted, Schema } from "effect";
+import { HttpServerResponse } from "effect/unstable/http";
+import { createHmac, timingSafeEqual } from "node:crypto";
+
+import type { ShopifyConfig } from "./manifest";
+
+import * as ShopifyApiClient from "./api";
+import * as ShopifyAuth from "./auth";
+import {
+  CartEventSchema,
+  CartWebhookPayloadSchema,
+  ProductDeleteWebhookPayloadSchema,
+  ProductEventSchema,
+  ProductSchema,
+  ProductWebhookPayloadSchema,
+  ShopifyNormalize,
+} from "./schemas";
+export { manifest, ShopifyConfigDef } from "./manifest";
+export type { ShopifyConfig } from "./manifest";
+
+const verifyWebhookSignature = (options: {
+  readonly rawBody: Uint8Array;
+  readonly signature: string | null;
+  readonly secret: string;
+}): Effect.Effect<void, ConnectorError> =>
+  Effect.try({
+    try: () => {
+      if (!options.signature) {
+        throw new Error("Missing x-shopify-hmac-sha256 header");
+      }
+      const digest = createHmac("sha256", options.secret)
+        .update(Buffer.from(options.rawBody))
+        .digest();
+      const provided = Buffer.from(options.signature, "base64");
+      if (provided.length !== digest.length || !timingSafeEqual(digest, provided)) {
+        throw new Error("Invalid Shopify webhook signature");
+      }
+    },
+    catch: (cause) =>
+      new ConnectorError({
+        message: "Shopify webhook verification failed",
+        cause,
+      }),
+  });
+
+const decodeWebhookPayload = <A>(schema: Schema.Decoder<A>) => Schema.decodeUnknownEffect(schema);
+
+export const make = Effect.fnUntraced(function* (config: ShopifyConfig) {
+  const api = yield* ShopifyApiClient.ShopifyApiClient;
+
+  const Products = Resource.entity({
+    name: "products",
+    schema: ProductSchema,
+    key: "id",
+    version: "updatedAt",
+    check: api.checkProductsAccess,
+    backfill: Fetch.page({
+      pageCursor: Cursor.string(),
+      cutoff: Cursor.isoDateTime(),
+      fetch: ({ pageCursor, cutoff }) =>
+        api
+          .fetchProducts({
+            first: 50,
+            after: typeof pageCursor === "string" ? pageCursor : undefined,
+          })
+          .pipe(
+            Effect.map((page) => ({
+              mutations: page.items
+                .filter((row) => Date.parse(row.updatedAt) <= Date.parse(String(cutoff)))
+                .map(Resource.upsert),
+              nextPageCursor: page.endCursor ?? undefined,
+              hasMore: page.hasMore,
+            })),
+          ),
+    }),
+    webhook: Resource.webhook({
+      schema: ProductEventSchema,
+      handler: ({ payload }) => {
+        if (payload._tag === "delete") {
+          return Effect.succeed([
+            Resource.delete({
+              key: `gid://shopify/Product/${payload.id}`,
+              version: payload.version,
+            }),
+          ]);
+        }
+        const product = payload.payload;
+        if (
+          product.created_at !== null &&
+          product.variants.length === product.variant_gids.length
+        ) {
+          return Effect.succeed([
+            Resource.upsert(
+              ShopifyNormalize.productWebhook({ ...product, created_at: product.created_at }),
+            ),
+          ]);
+        }
+        // Refetch when Shopify truncates variants or omits the creation time.
+        return api
+          .fetchProductById(product.admin_graphql_api_id)
+          .pipe(Effect.map((row) => [Resource.upsert(row)]));
+      },
+    }),
+  });
+
+  const CartEvents = Resource.entity({
+    name: "cart_events",
+    schema: CartEventSchema,
+    key: "id",
+    version: "updatedAt",
+    check: api.checkConnection,
+    webhook: Resource.webhook({
+      schema: Schema.Struct({
+        ...CartWebhookPayloadSchema.fields,
+        topic: Schema.Literals(["carts/create", "carts/update"]),
+      }),
+      handler: ({ payload }) =>
+        Effect.succeed([Resource.upsert(ShopifyNormalize.cartWebhook(payload, payload.topic))]),
+    }),
+  });
+
+  const webhookRoute = Webhook.route({
+    path: "/webhooks/shopify",
+    ackMode: "after-publish",
+    schema: Schema.Unknown,
+    handler: ({ request, rawBody, payload: json, to }) =>
+      Effect.gen(function* () {
+        const verificationError = yield* verifyWebhookSignature({
+          rawBody,
+          signature: request.headers["x-shopify-hmac-sha256"] ?? null,
+          secret: Redacted.value(config.webhookSecret),
+        }).pipe(Effect.match({ onFailure: (error) => error, onSuccess: () => undefined }));
+        if (verificationError) {
+          return HttpServerResponse.jsonUnsafe(
+            { ok: false, error: verificationError.message },
+            { status: 401 },
+          );
+        }
+
+        const topic = request.headers["x-shopify-topic"] ?? "";
+        switch (topic) {
+          case "products/create":
+          case "products/update": {
+            const payload = yield* decodeWebhookPayload(ProductWebhookPayloadSchema)(json).pipe(
+              Effect.mapError(
+                (cause) =>
+                  new ConnectorError({
+                    message: `Invalid Shopify webhook payload for ${topic}`,
+                    cause,
+                  }),
+              ),
+              Effect.match({
+                onFailure: (error) => ({ _tag: "Error" as const, error }),
+                onSuccess: (value) => ({ _tag: "Success" as const, value }),
+              }),
+            );
+            if (payload._tag === "Error") {
+              return HttpServerResponse.jsonUnsafe(
+                { ok: false, error: payload.error.message },
+                { status: 400 },
+              );
+            }
+            yield* to(Products, { _tag: "upsert", payload: payload.value });
+            break;
+          }
+          case "products/delete": {
+            const triggeredAt = request.headers["x-shopify-triggered-at"];
+            if (!triggeredAt) {
+              return HttpServerResponse.jsonUnsafe(
+                { ok: false, error: "Missing x-shopify-triggered-at header" },
+                { status: 400 },
+              );
+            }
+            const payload = yield* decodeWebhookPayload(ProductDeleteWebhookPayloadSchema)(
+              json,
+            ).pipe(
+              Effect.mapError(
+                (cause) =>
+                  new ConnectorError({
+                    message: "Invalid Shopify webhook payload for products/delete",
+                    cause,
+                  }),
+              ),
+              Effect.match({
+                onFailure: (error) => ({ _tag: "Error" as const, error }),
+                onSuccess: (value) => ({ _tag: "Success" as const, value }),
+              }),
+            );
+            if (payload._tag === "Error") {
+              return HttpServerResponse.jsonUnsafe(
+                { ok: false, error: payload.error.message },
+                { status: 400 },
+              );
+            }
+            yield* to(Products, {
+              _tag: "delete",
+              id: String(payload.value.id),
+              version: triggeredAt,
+            });
+            break;
+          }
+          case "carts/create":
+          case "carts/update": {
+            const payload = yield* decodeWebhookPayload(CartWebhookPayloadSchema)(json).pipe(
+              Effect.mapError(
+                (cause) =>
+                  new ConnectorError({
+                    message: `Invalid Shopify webhook payload for ${topic}`,
+                    cause,
+                  }),
+              ),
+              Effect.match({
+                onFailure: (error) => ({ _tag: "Error" as const, error }),
+                onSuccess: (value) => ({ _tag: "Success" as const, value }),
+              }),
+            );
+            if (payload._tag === "Error") {
+              return HttpServerResponse.jsonUnsafe(
+                { ok: false, error: payload.error.message },
+                { status: 400 },
+              );
+            }
+            yield* to(CartEvents, { ...payload.value, topic });
+            break;
+          }
+          default:
+            yield* Effect.logWarning("Ignoring unknown Shopify webhook topic").pipe(
+              Effect.annotateLogs({ topic }),
+            );
+        }
+
+        return HttpServerResponse.jsonUnsafe({ ok: true });
+      }),
+  });
+
+  return Connector.define({
+    name: "producer-shopify",
+    title: "Shopify",
+    resources: [Products, CartEvents],
+    webhooks: [webhookRoute],
+  });
+});
+
+export type ShopifyConnectorRuntime = Effect.Success<ReturnType<typeof make>>;
+
+export class ShopifyConnector extends Context.Service<ShopifyConnector, ShopifyConnectorRuntime>()(
+  "@useairfoil/producer-shopify/ShopifyConnector",
+) {}
+
+export const layer = (config: ShopifyConfig) => {
+  const authLayer = ShopifyAuth.layer(config);
+  const apiLayer = ShopifyApiClient.layer(config).pipe(Layer.provide(authLayer));
+
+  return Layer.effect(ShopifyConnector)(
+    make(config).pipe(Effect.annotateLogs({ component: "producer-shopify" })),
+  ).pipe(Layer.provide(apiLayer));
+};
+
+export const layerConfig = (config: Config.Wrap<ShopifyConfig>) =>
+  Layer.unwrap(Config.unwrap(config).pipe(Effect.map(layer)));

package/src/index.ts

@@ -0,0 +1,36 @@
+export * as ShopifyApiClient from "./api";
+export * as ShopifyConnector from "./connector";
+export { manifest } from "./manifest";
+export type {
+  CartEvent,
+  CartLineItem,
+  CartWebhookPayload,
+  Money,
+  MoneyBag,
+  PageInfo,
+  Product,
+  ProductDeleteWebhookPayload,
+  ProductOption,
+  ProductStatus,
+  ProductVariant,
+  ProductVariantInventoryPolicy,
+  ProductWebhookPayload,
+  WebhookPayload,
+} from "./schemas";
+export {
+  CartEventSchema,
+  CartLineItemSchema,
+  CartWebhookPayloadSchema,
+  MoneyBagSchema,
+  MoneySchema,
+  PageInfoSchema,
+  ProductOptionSchema,
+  ProductDeleteWebhookPayloadSchema,
+  ProductSchema,
+  ProductStatusSchema,
+  ProductVariantInventoryPolicySchema,
+  ProductVariantSchema,
+  ProductWebhookPayloadSchema,
+  ShopifyNormalize,
+  WebhookPayloadSchema,
+} from "./schemas";

package/src/main.ts

@@ -0,0 +1,20 @@
+import { NodeRuntime, NodeServices } from "@effect/platform-node";
+import { Effect, Layer } from "effect";
+import { Command } from "effect/unstable/cli";
+import { FetchHttpClient } from "effect/unstable/http";
+
+import packageJson from "../package.json";
+import { sandboxCommand } from "./sandbox";
+import { startCommand } from "./start";
+
+const BootstrapLayer = Layer.mergeAll(FetchHttpClient.layer, NodeServices.layer);
+
+const program = Command.make("producer-shopify", {}, () => Effect.void).pipe(
+  Command.withSubcommands([startCommand, sandboxCommand]),
+);
+
+Command.run(program, { version: packageJson.version }).pipe(
+  Effect.provide(BootstrapLayer),
+  Effect.scoped,
+  NodeRuntime.runMain,
+);

package/src/manifest.ts

@@ -0,0 +1,86 @@
+import * as Manifest from "@useairfoil/connector-kit/manifest";
+
+export const ShopifyConfigDef = Manifest.defineConfig({
+  shopDomain: Manifest.string({
+    runtimeKey: "SHOPIFY_SHOP_DOMAIN",
+    description: "Shopify shop domain, for example example.myshopify.com.",
+  }),
+  apiVersion: Manifest.string({
+    runtimeKey: "SHOPIFY_API_VERSION",
+    description: "Shopify Admin API version used in GraphQL endpoint URLs.",
+    default: "2026-07",
+  }),
+  clientId: Manifest.string({
+    runtimeKey: "SHOPIFY_CLIENT_ID",
+    description: "Client ID for the merchant-owned Shopify Dev Dashboard app.",
+  }),
+  clientSecret: Manifest.secret({
+    runtimeKey: "SHOPIFY_CLIENT_SECRET",
+    description: "Client secret for the merchant-owned Shopify Dev Dashboard app.",
+  }),
+  responseMaxRetries: Manifest.number({
+    runtimeKey: "SHOPIFY_RESPONSE_MAX_RETRIES",
+    description: "Maximum retries for temporary HTTP responses.",
+    default: 5,
+    integer: true,
+    minimum: 0,
+  }),
+  transportMaxRetries: Manifest.number({
+    runtimeKey: "SHOPIFY_TRANSPORT_MAX_RETRIES",
+    description: "Maximum retries for connection and transport failures.",
+    default: 5,
+    integer: true,
+    minimum: 0,
+  }),
+  graphqlMaxRetries: Manifest.number({
+    runtimeKey: "SHOPIFY_GRAPHQL_MAX_RETRIES",
+    description: "Maximum retries for temporary GraphQL errors.",
+    default: 5,
+    integer: true,
+    minimum: 0,
+  }),
+  retryBaseDelayMs: Manifest.number({
+    runtimeKey: "SHOPIFY_RETRY_BASE_DELAY_MS",
+    description: "Initial delay for HTTP and transport retries.",
+    default: 200,
+    integer: true,
+    minimum: 1,
+  }),
+  graphqlRetryBaseDelayMs: Manifest.number({
+    runtimeKey: "SHOPIFY_GRAPHQL_RETRY_BASE_DELAY_MS",
+    description: "Initial delay for temporary GraphQL errors.",
+    default: 500,
+    integer: true,
+    minimum: 1,
+  }),
+  retryAfterFallbackSeconds: Manifest.number({
+    runtimeKey: "SHOPIFY_RETRY_AFTER_FALLBACK_SECONDS",
+    description: "Retry delay used when Shopify omits Retry-After.",
+    default: 1,
+    integer: true,
+    minimum: 1,
+  }),
+  requestTimeoutSeconds: Manifest.number({
+    runtimeKey: "SHOPIFY_REQUEST_TIMEOUT_SECONDS",
+    description: "Maximum request time including retries.",
+    default: 120,
+    integer: true,
+    minimum: 1,
+  }),
+  webhookSecret: Manifest.secret({
+    runtimeKey: "SHOPIFY_WEBHOOK_SECRET",
+    description: "Shopify webhook HMAC secret.",
+  }),
+});
+
+export type ShopifyConfig = Manifest.ConfigValuesOf<typeof ShopifyConfigDef>;
+
+export const manifest = Manifest.define({
+  name: "producer-shopify",
+  title: "Shopify",
+  config: ShopifyConfigDef.spec,
+  resources: [
+    { name: "products", capabilities: ["backfill", "webhook"] },
+    { name: "cart_events", capabilities: ["webhook"] },
+  ],
+});

package/src/sandbox.ts

@@ -0,0 +1,38 @@
+import {
+  ConnectorApp,
+  Publisher,
+  RuntimeConfig,
+  StateStore,
+  Telemetry,
+} from "@useairfoil/connector-kit";
+import { Effect, Layer, Logger } from "effect";
+import { Command } from "effect/unstable/cli";
+
+import { ShopifyConnector } from "./index";
+
+const ConnectorLayer = ShopifyConnector.layerConfig(ShopifyConnector.ShopifyConfigDef.config);
+
+const TelemetryLayer = Layer.mergeAll(
+  Telemetry.layerOtlp({ redactedHeaders: ["x-shopify-access-token"] }),
+  Telemetry.layerMetricsConsoleDump(),
+);
+
+export const sandboxCommand = Command.make("sandbox", {}, () =>
+  Effect.gen(function* () {
+    const port = yield* RuntimeConfig.httpPort;
+    const entrypoint = yield* ShopifyConnector.ShopifyConnector;
+
+    return yield* ConnectorApp.start(entrypoint, { port });
+  }).pipe(
+    Effect.annotateLogs({ component: "producer-shopify" }),
+    Effect.provide(
+      Layer.mergeAll(
+        StateStore.layerMemory,
+        Publisher.layerConsole,
+        ConnectorLayer,
+        Logger.layer([Logger.consolePretty()]),
+        TelemetryLayer,
+      ),
+    ),
+  ),
+).pipe(Command.withDescription("Run the connector locally and log ingested data"));