@reactionary/feeds 0.0.1

package/lib/feed-server.js

@@ -0,0 +1,305 @@
+import {
+  createInitialRequestContext
+} from "@reactionary/core";
+import { createDefaultFeedRegistry } from "./default-registry.js";
+import { ReactionaryFeedGenerator } from "./feed-generator.js";
+import { ReactionarySitemapGenerator } from "./sitemap-generator.js";
+import {
+  toSitemapIndexXml,
+  toSitemapXml
+} from "./sitemap-xml.js";
+class ReactionaryFeedServer {
+  constructor(clientFactory, options) {
+    this.clientFactory = clientFactory;
+    this.options = options;
+    this.registry = options.registry ?? createDefaultFeedRegistry();
+  }
+  clientFactory;
+  options;
+  registry;
+  async fetch(request) {
+    try {
+      return await this.handleRequest(request);
+    } catch (error) {
+      return jsonResponse({
+        error: error instanceof Error ? error.message : String(error)
+      }, { status: 500 });
+    }
+  }
+  getHandler() {
+    return {
+      fetch: (request) => this.fetch(request),
+      close: () => this.close()
+    };
+  }
+  toNodeHandler() {
+    return async (request, response) => {
+      const webResponse = await this.fetch(await toWebRequest(request));
+      await sendWebResponse(response, webResponse);
+    };
+  }
+  close() {
+    return Promise.resolve();
+  }
+  async handleRequest(request) {
+    if (request.method !== "GET" && request.method !== "HEAD") {
+      return jsonResponse({
+        error: `Unsupported method: ${request.method}`
+      }, {
+        status: 405,
+        headers: { allow: "GET, HEAD" }
+      });
+    }
+    const route = parseFeedRoute(request, this.options.basePath);
+    const sitemapRoute = parseSitemapRoute(request, this.options.sitemaps);
+    if (sitemapRoute.kind === "index") {
+      return this.generateSitemapIndexResponse(request);
+    }
+    if (sitemapRoute.kind === "source") {
+      return this.generateSitemapResponse(request, sitemapRoute.sourceId);
+    }
+    if (route.kind === "feeds") {
+      return jsonResponse({
+        feeds: Object.keys(this.options.feeds)
+      }, { omitBody: request.method === "HEAD" });
+    }
+    if (route.kind === "transformers") {
+      return jsonResponse({
+        transformers: this.registry.list()
+      }, { omitBody: request.method === "HEAD" });
+    }
+    if (route.kind !== "output") {
+      return jsonResponse({
+        error: "Not found"
+      }, { status: 404, omitBody: request.method === "HEAD" });
+    }
+    const feed = this.options.feeds[route.feedId];
+    const transformer = this.registry.get(route.transformerId);
+    if (!feed) {
+      return jsonResponse({
+        error: `Feed not found: ${route.feedId}`
+      }, { status: 404 });
+    }
+    if (!transformer) {
+      return jsonResponse({
+        error: `Feed transformer not found: ${route.transformerId}`
+      }, { status: 404 });
+    }
+    return this.generateFeedResponse(
+      request,
+      route.feedId,
+      feed,
+      transformer
+    );
+  }
+  generateFeedResponse(request, feedId, feed, transformer) {
+    const requestContext = createInitialRequestContext();
+    requestContext.languageContext = feed.languageContext;
+    const client = this.clientFactory(requestContext);
+    const generator = new ReactionaryFeedGenerator(client, {
+      defaultFulfillmentCenterKeys: this.options.defaultFulfillmentCenterKeys,
+      productConcurrency: this.options.productConcurrency
+    });
+    const output = transformer.transform(
+      generator.products(feed, requestContext),
+      {
+        feedId,
+        feed,
+        options: transformer.defaultOptions
+      }
+    );
+    return new Response(
+      request.method === "HEAD" ? null : iterableToStream(output),
+      {
+        headers: {
+          "content-type": transformer.output.contentType,
+          "content-disposition": `attachment; filename="${feedId}.${transformer.output.fileExtension}"`
+        }
+      }
+    );
+  }
+  async generateSitemapIndexResponse(request) {
+    const sitemaps = this.options.sitemaps;
+    if (!sitemaps) {
+      return jsonResponse({
+        error: "Sitemaps are not configured."
+      }, { status: 404, omitBody: request.method === "HEAD" });
+    }
+    const baseUrl = trimTrailingSlash(sitemaps.baseUrl);
+    const sourceIds = getIncludedSitemapSourceIds(sitemaps);
+    const xml = await toSitemapIndexXml(
+      sourceIds.map((sourceId) => `${baseUrl}/sitemaps/${encodeURIComponent(sourceId)}.xml`)
+    );
+    return xmlResponse(xml, {
+      omitBody: request.method === "HEAD"
+    });
+  }
+  async generateSitemapResponse(request, sourceId) {
+    const sitemaps = this.options.sitemaps;
+    if (!sitemaps) {
+      return jsonResponse({
+        error: "Sitemaps are not configured."
+      }, { status: 404, omitBody: request.method === "HEAD" });
+    }
+    if (!getIncludedSitemapSourceIds(sitemaps).includes(sourceId)) {
+      return jsonResponse({
+        error: `Sitemap source not found: ${sourceId}`
+      }, { status: 404, omitBody: request.method === "HEAD" });
+    }
+    const source = sitemaps.sources[sourceId];
+    if (!source) {
+      return jsonResponse({
+        error: `Sitemap source not found: ${sourceId}`
+      }, { status: 404, omitBody: request.method === "HEAD" });
+    }
+    const requestContext = createInitialRequestContext();
+    const languageContext = source.type === "products" ? this.options.feeds[source.feed]?.languageContext : source.languageContext;
+    if (languageContext) {
+      requestContext.languageContext = languageContext;
+    }
+    const client = this.clientFactory(requestContext);
+    const generator = new ReactionarySitemapGenerator(
+      client,
+      this.options.feeds,
+      {
+        defaultFulfillmentCenterKeys: this.options.defaultFulfillmentCenterKeys,
+        productConcurrency: this.options.productConcurrency
+      }
+    );
+    const xml = await toSitemapXml(generator.entries(source, requestContext));
+    return xmlResponse(xml, {
+      headers: {
+        "content-disposition": `attachment; filename="${sourceId}.xml"`
+      },
+      omitBody: request.method === "HEAD"
+    });
+  }
+}
+function parseSitemapRoute(request, sitemaps) {
+  if (!sitemaps) {
+    return { kind: "not-found" };
+  }
+  const pathname = new URL(request.url).pathname;
+  if (pathname === "/sitemaps.xml") {
+    return { kind: "index" };
+  }
+  const match = /^\/sitemaps\/([^/]+)\.xml$/.exec(pathname);
+  if (match) {
+    return {
+      kind: "source",
+      sourceId: decodeURIComponent(match[1] ?? "")
+    };
+  }
+  return { kind: "not-found" };
+}
+function parseFeedRoute(request, basePath = "/feeds") {
+  const pathname = getProtocolPathname(request, basePath);
+  if (pathname === "/" || pathname === "") {
+    return { kind: "feeds" };
+  }
+  if (pathname === "/transformers") {
+    return { kind: "transformers" };
+  }
+  const match = /^\/([^/]+)\/outputs\/([^/]+)$/.exec(pathname);
+  if (match) {
+    return {
+      kind: "output",
+      feedId: decodeURIComponent(match[1] ?? ""),
+      transformerId: decodeURIComponent(match[2] ?? "")
+    };
+  }
+  return { kind: "not-found" };
+}
+function getProtocolPathname(request, basePath) {
+  const pathname = new URL(request.url).pathname;
+  if (pathname === basePath) {
+    return "/";
+  }
+  if (pathname.startsWith(`${basePath}/`)) {
+    return pathname.slice(basePath.length);
+  }
+  return pathname;
+}
+function iterableToStream(iterable) {
+  const encoder = new TextEncoder();
+  const iterator = iterable[Symbol.asyncIterator]();
+  return new ReadableStream({
+    async pull(controller) {
+      const next = await iterator.next();
+      if (next.done) {
+        controller.close();
+        return;
+      }
+      controller.enqueue(
+        typeof next.value === "string" ? encoder.encode(next.value) : next.value
+      );
+    },
+    async cancel() {
+      await iterator.return?.();
+    }
+  });
+}
+function jsonResponse(body, options = {}) {
+  return new Response(options.omitBody ? null : JSON.stringify(body), {
+    status: options.status ?? 200,
+    headers: {
+      "content-type": "application/json; charset=utf-8",
+      ...options.headers
+    }
+  });
+}
+function xmlResponse(body, options = {}) {
+  return new Response(options.omitBody ? null : body, {
+    headers: {
+      "content-type": "application/xml; charset=utf-8",
+      ...options.headers
+    }
+  });
+}
+function getIncludedSitemapSourceIds(sitemaps) {
+  return sitemaps.include ?? Object.keys(sitemaps.sources);
+}
+function trimTrailingSlash(value) {
+  return value.replace(/\/+$/, "");
+}
+async function toWebRequest(request) {
+  const headers = toWebHeaders(request.headers);
+  const url = new URL(
+    request.url ?? "/",
+    `http://${request.headers.host ?? "localhost"}`
+  );
+  return new Request(url, {
+    method: request.method,
+    headers
+  });
+}
+function toWebHeaders(headers) {
+  const webHeaders = new Headers();
+  for (const [key, value] of Object.entries(headers)) {
+    if (value === void 0) {
+      continue;
+    }
+    if (Array.isArray(value)) {
+      for (const entry of value) {
+        webHeaders.append(key, entry);
+      }
+      continue;
+    }
+    webHeaders.set(key, value);
+  }
+  return webHeaders;
+}
+async function sendWebResponse(response, webResponse) {
+  response.writeHead(
+    webResponse.status,
+    Object.fromEntries(webResponse.headers.entries())
+  );
+  if (webResponse.body) {
+    response.end(Buffer.from(await webResponse.arrayBuffer()));
+    return;
+  }
+  response.end();
+}
+export {
+  ReactionaryFeedServer
+};

package/lib/feed-transformer.js

@@ -0,0 +1,21 @@
+class ReactionaryFeedTransformerRegistry {
+  transformers = /* @__PURE__ */ new Map();
+  register(transformer) {
+    this.transformers.set(transformer.id, transformer);
+  }
+  get(id) {
+    return this.transformers.get(id);
+  }
+  list() {
+    return [...this.transformers.values()].map((transformer) => ({
+      id: transformer.id,
+      title: transformer.title,
+      description: transformer.description,
+      contentType: transformer.output.contentType,
+      fileExtension: transformer.output.fileExtension
+    }));
+  }
+}
+export {
+  ReactionaryFeedTransformerRegistry
+};

package/lib/feed-types.js

@@ -0,0 +1,45 @@
+function assertReactionaryFeedClient(client) {
+  const missing = getMissingFeedClientOperations(client);
+  if (missing.length > 0) {
+    throw new Error(
+      `Reactionary feed generator cannot initialize because the client is missing required operations: ${missing.join(", ")}`
+    );
+  }
+}
+function getMissingFeedClientOperations(client) {
+  return [
+    ["productSearch.queryByTerm", client.productSearch?.queryByTerm],
+    ["product.getBySKU", client.product?.getBySKU],
+    ["price.getCustomerPrice", client.price?.getCustomerPrice],
+    ["price.getListPrice", client.price?.getListPrice],
+    ["inventory.getBySKU", client.inventory?.getBySKU]
+  ].filter(([, operation]) => typeof operation !== "function").map(([name]) => name);
+}
+function getMissingSitemapClientOperations(client, source) {
+  if (source.type === "products") {
+    return getMissingFeedClientOperations(client);
+  }
+  if (source.type === "categories") {
+    return [
+      ["category.findTopCategories", client.category?.findTopCategories],
+      ["category.findChildCategories", client.category?.findChildCategories]
+    ].filter(([, operation]) => typeof operation !== "function").map(([name]) => name);
+  }
+  return [
+    ["store.queryByProximity", client.store?.queryByProximity]
+  ].filter(([, operation]) => typeof operation !== "function").map(([name]) => name);
+}
+function assertReactionarySitemapClient(client, source) {
+  const missing = getMissingSitemapClientOperations(client, source);
+  if (missing.length > 0) {
+    throw new Error(
+      `Reactionary sitemap source cannot initialize because the client is missing required operations: ${missing.join(", ")}`
+    );
+  }
+}
+export {
+  assertReactionaryFeedClient,
+  assertReactionarySitemapClient,
+  getMissingFeedClientOperations,
+  getMissingSitemapClientOperations
+};

package/lib/load-cwd-env.js

@@ -0,0 +1,16 @@
+import { config } from "dotenv";
+function loadCwdEnv(options = {}) {
+  const env = options.env ?? process.env;
+  const result = env["DOTENV_CONFIG_PATH"] ? config({ path: env["DOTENV_CONFIG_PATH"] }) : config();
+  if (!result.error) {
+    return;
+  }
+  const errorCode = result.error.code;
+  if (errorCode === "ENOENT" && env["DOTENV_CONFIG_PATH"] === void 0) {
+    return;
+  }
+  throw result.error;
+}
+export {
+  loadCwdEnv
+};

package/lib/sitemap-generator.js

@@ -0,0 +1,173 @@
+import { ReactionaryFeedGenerator } from "./feed-generator.js";
+import {
+  assertReactionarySitemapClient
+} from "./feed-types.js";
+class ReactionarySitemapGenerator {
+  constructor(client, feeds, options = {}) {
+    this.client = client;
+    this.feeds = feeds;
+    this.options = options;
+  }
+  client;
+  feeds;
+  options;
+  async *entries(source, requestContext) {
+    assertReactionarySitemapClient(this.client, source);
+    if (source.type === "products") {
+      yield* this.productEntries(source.feed, source, requestContext);
+      return;
+    }
+    requestContext.languageContext = source.languageContext;
+    if (source.type === "categories") {
+      yield* this.categoryEntries(source);
+      return;
+    }
+    yield* this.storeEntries(source);
+  }
+  async *productEntries(feedId, source, requestContext) {
+    const feed = this.feeds[feedId];
+    if (!feed) {
+      throw new Error(`Product sitemap feed not found: ${feedId}`);
+    }
+    const generator = new ReactionaryFeedGenerator(this.client, this.options);
+    for await (const product of generator.products(feed, requestContext)) {
+      if (!product.url) {
+        continue;
+      }
+      yield {
+        url: product.url,
+        changefreq: source.changefreq,
+        priority: source.priority
+      };
+    }
+  }
+  async *categoryEntries(source) {
+    const client = this.client;
+    if (!client.category) {
+      return;
+    }
+    const topCategories = this.iterCategoryPageResults(
+      (pageNumber, pageSize) => client.category?.findTopCategories({
+        paginationOptions: {
+          pageNumber,
+          pageSize
+        }
+      }) ?? Promise.resolve({
+        success: true,
+        value: emptyCategoryPage(pageNumber, pageSize)
+      }),
+      source
+    );
+    for await (const category of topCategories) {
+      yield* this.categoryWithChildrenEntries(category, source, 1);
+    }
+  }
+  async *categoryWithChildrenEntries(category, source, depth) {
+    const url = toTemplatedUrl(source.urlTemplate, source.languageContext, {
+      id: category.identifier.key,
+      slug: category.slug || category.identifier.key
+    });
+    if (url) {
+      yield {
+        url,
+        changefreq: source.changefreq,
+        priority: source.priority
+      };
+    }
+    if (source.includeChildren === false || depth >= (source.maxDepth ?? 10)) {
+      return;
+    }
+    const client = this.client;
+    if (!client.category) {
+      return;
+    }
+    const childCategories = this.iterCategoryPageResults(
+      (pageNumber, pageSize) => client.category?.findChildCategories({
+        parentId: category.identifier,
+        paginationOptions: {
+          pageNumber,
+          pageSize
+        }
+      }) ?? Promise.resolve({
+        success: true,
+        value: emptyCategoryPage(pageNumber, pageSize)
+      }),
+      source
+    );
+    for await (const child of childCategories) {
+      yield* this.categoryWithChildrenEntries(child, source, depth + 1);
+    }
+  }
+  async *iterCategoryPageResults(loadPage, source) {
+    const pageSize = source.pageSize ?? 50;
+    const maxPages = source.maxPagesPerLevel ?? 100;
+    let pageNumber = 1;
+    while (pageNumber <= maxPages) {
+      const result = await loadPage(pageNumber, pageSize);
+      if (!result.success) {
+        throw new Error(JSON.stringify(result.error));
+      }
+      for (const category of result.value.items) {
+        yield category;
+      }
+      if (pageNumber >= result.value.totalPages || result.value.items.length === 0) {
+        return;
+      }
+      pageNumber += 1;
+    }
+  }
+  async *storeEntries(source) {
+    const result = await this.client.store?.queryByProximity(source.proximity);
+    if (!result) {
+      return;
+    }
+    if (!result.success) {
+      throw new Error(JSON.stringify(result.error));
+    }
+    for (const store of result.value) {
+      const url = toTemplatedUrl(source.urlTemplate, source.languageContext, {
+        id: store.identifier.key,
+        slug: slugifyStore(store)
+      });
+      if (!url) {
+        continue;
+      }
+      yield {
+        url,
+        changefreq: source.changefreq,
+        priority: source.priority
+      };
+    }
+  }
+}
+function toTemplatedUrl(urlTemplate, languageContext, values) {
+  const slug = values.slug ?? values.id;
+  if (!urlTemplate) {
+    return void 0;
+  }
+  if (urlTemplate.includes("{lang}") || urlTemplate.includes("{slug}") || urlTemplate.includes("{id}")) {
+    return urlTemplate.replaceAll("{lang}", getLanguage(languageContext)).replaceAll("{slug}", encodeURIComponent(slug)).replaceAll("{id}", encodeURIComponent(values.id));
+  }
+  return new URL(slug, urlTemplate).href;
+}
+function getLanguage(languageContext) {
+  const [language] = languageContext.locale.split("-");
+  return language.toLowerCase();
+}
+function slugifyStore(store) {
+  const slug = store.name.trim().toLowerCase().replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-|-$/g, "");
+  return slug || store.identifier.key;
+}
+function emptyCategoryPage(pageNumber, pageSize) {
+  return {
+    pageNumber,
+    pageSize,
+    totalCount: 0,
+    totalPages: 0,
+    items: []
+  };
+}
+export {
+  ReactionarySitemapGenerator,
+  toTemplatedUrl
+};

package/lib/sitemap-xml.js

@@ -0,0 +1,32 @@
+import {
+  SitemapIndexStream,
+  SitemapStream,
+  streamToPromise
+} from "sitemap";
+async function toSitemapXml(entries) {
+  const sitemap = new SitemapStream();
+  for await (const entry of entries) {
+    sitemap.write({
+      url: entry.url,
+      lastmod: entry.lastmod,
+      changefreq: entry.changefreq,
+      priority: entry.priority
+    });
+  }
+  sitemap.end();
+  return (await streamToPromise(sitemap)).toString();
+}
+async function toSitemapIndexXml(urls) {
+  const sitemapIndex = new SitemapIndexStream();
+  for (const url of urls) {
+    sitemapIndex.write({
+      url
+    });
+  }
+  sitemapIndex.end();
+  return (await streamToPromise(sitemapIndex)).toString();
+}
+export {
+  toSitemapIndexXml,
+  toSitemapXml
+};