@reactionary/feeds 0.0.1

package/cli.js

@@ -0,0 +1,340 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { createWriteStream } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { finished } from "node:stream/promises";
+import { pathToFileURL } from "node:url";
+import { createDefaultFeedRegistry } from "./lib/default-registry.js";
+import { ReactionaryFeedGenerator } from "./lib/feed-generator.js";
+async function main() {
+  const program = new Command().name("reactionary-feeds").description("Generate product feeds from Reactionary clients.").version("0.0.1");
+  program.command("list-feeds").description("List feed ids from a feed config module.").requiredOption("-c, --config <path>", "Path to an ESM feed config module.").action(async (options) => {
+    const config = await loadConfig(options.config);
+    console.log(JSON.stringify({ feeds: Object.keys(config.feeds) }, null, 2));
+  });
+  program.command("list-transformers").description("List built-in feed output transformers.").requiredOption("-c, --config <path>", "Path to an ESM feed config module.").action(async (options) => {
+    await loadConfig(options.config);
+    const registry = createDefaultFeedRegistry();
+    console.log(JSON.stringify({ transformers: registry.list() }, null, 2));
+  });
+  program.command("generate").description("Generate one or more feed outputs in a single source pass.").requiredOption("-c, --config <path>", "Path to an ESM feed config module.").requiredOption("-f, --feed <id>", "Configured feed id to generate.").requiredOption(
+    "-o, --output <transformer=path>",
+    "Transformer/output pair. Repeat for each output file.",
+    collectOption,
+    []
+  ).option("--product-concurrency <count>", "Products to normalize in parallel. Defaults to config value or 10.").option("--testMode", "Limit source reads to at most 2 pages for smoke testing.").option("--no-progress", "Disable progress reporting on stderr.").action(async (options) => {
+    await generateFeed(options);
+  });
+  await program.parseAsync(process.argv);
+}
+async function generateFeed(options) {
+  const context = await createFeedGenerationContext(options, {
+    feedId: options.feed,
+    transformerLabel: "outputs"
+  });
+  const outputs = parseOutputSpecs(options.output, context.registry);
+  const productOutputs = fanoutAsyncIterable(
+    context.generator.products(context.feed, context.requestContext),
+    outputs.length
+  );
+  await Promise.all(outputs.map((output, index) => {
+    const products = productOutputs[index];
+    if (!products) {
+      throw new Error(`Missing product stream for output: ${output.transformerId}`);
+    }
+    return writeTransformerOutput({
+      feedId: options.feed,
+      feed: context.feed,
+      output,
+      products
+    });
+  }));
+}
+async function createFeedGenerationContext(options, labels) {
+  const config = await loadConfig(options.config);
+  const registry = createDefaultFeedRegistry();
+  const configuredFeed = config.feeds[labels.feedId];
+  if (!configuredFeed) {
+    throw new Error(`Feed not found: ${labels.feedId}`);
+  }
+  const feed = options.testMode ? withTestModeMaxPages(configuredFeed) : configuredFeed;
+  const requestContext = createCliRequestContext();
+  requestContext.languageContext = feed.languageContext;
+  const createClient = await getClientFactory(config);
+  const client = createClient(requestContext);
+  const progressReporter = options.progress === false ? void 0 : createCliProgressReporter({
+    feedId: labels.feedId,
+    transformerId: labels.transformerLabel,
+    stream: process.stderr
+  });
+  const generator = new ReactionaryFeedGenerator(client, {
+    defaultFulfillmentCenterKeys: config.defaultFulfillmentCenterKeys,
+    productConcurrency: parseProductConcurrency(
+      options.productConcurrency,
+      config.productConcurrency
+    ),
+    onProgress: progressReporter
+  });
+  return {
+    config,
+    feed,
+    registry,
+    requestContext,
+    generator
+  };
+}
+function withTestModeMaxPages(feed) {
+  return {
+    ...feed,
+    maxPages: Math.min(feed.maxPages ?? 2, 2)
+  };
+}
+async function writeTransformerOutput(input) {
+  await mkdir(dirname(resolve(input.output.outputPath)), { recursive: true });
+  const stream = createWriteStream(resolve(input.output.outputPath));
+  try {
+    for await (const chunk of input.output.transformer.transform(
+      input.products,
+      {
+        feedId: input.feedId,
+        feed: input.feed,
+        options: input.output.transformer.defaultOptions
+      }
+    )) {
+      if (!stream.write(chunk)) {
+        await onceDrain(stream);
+      }
+    }
+  } finally {
+    stream.end();
+  }
+  await finished(stream);
+}
+function parseOutputSpecs(values, registry) {
+  if (values.length === 0) {
+    throw new Error("At least one --output transformer=path pair is required.");
+  }
+  const seen = /* @__PURE__ */ new Set();
+  return values.map((value) => {
+    const separator = value.indexOf("=");
+    if (separator <= 0 || separator === value.length - 1) {
+      throw new Error(`Invalid --output value "${value}". Expected transformer=path.`);
+    }
+    const transformerId = value.slice(0, separator);
+    const outputPath = value.slice(separator + 1);
+    if (seen.has(transformerId)) {
+      throw new Error(`Duplicate transformer output: ${transformerId}`);
+    }
+    seen.add(transformerId);
+    const transformer = registry.get(transformerId);
+    if (!transformer) {
+      throw new Error(`Transformer not found: ${transformerId}`);
+    }
+    return {
+      transformerId,
+      outputPath,
+      transformer
+    };
+  });
+}
+function fanoutAsyncIterable(source, consumerCount) {
+  const maxQueuedItemsPerConsumer = 20;
+  const queues = Array.from({ length: consumerCount }, () => []);
+  const waiters = Array.from({ length: consumerCount }, () => []);
+  const capacityWaiters = [];
+  let done = false;
+  let error = void 0;
+  const producer = (async () => {
+    try {
+      for await (const item of source) {
+        while (queues.some((queue) => queue.length >= maxQueuedItemsPerConsumer)) {
+          await new Promise((resolve2) => {
+            capacityWaiters.push(resolve2);
+          });
+        }
+        for (const queue of queues) {
+          queue.push(item);
+        }
+        for (const queueWaiters of waiters) {
+          queueWaiters.splice(0).forEach((wake) => wake());
+        }
+      }
+    } catch (caught) {
+      error = caught;
+    } finally {
+      done = true;
+      for (const queueWaiters of waiters) {
+        queueWaiters.splice(0).forEach((wake) => wake());
+      }
+    }
+  })();
+  return queues.map((queue, index) => ({
+    async *[Symbol.asyncIterator]() {
+      while (true) {
+        if (queue.length > 0) {
+          const item = queue.shift();
+          capacityWaiters.splice(0).forEach((wake) => wake());
+          yield item;
+          continue;
+        }
+        if (done) {
+          if (error) {
+            throw error;
+          }
+          await producer;
+          return;
+        }
+        await new Promise((resolve2) => {
+          waiters[index].push(resolve2);
+        });
+      }
+    }
+  }));
+}
+function collectOption(value, previous) {
+  return [...previous, value];
+}
+async function onceDrain(stream) {
+  await new Promise((resolve2, reject) => {
+    stream.once("drain", resolve2);
+    stream.once("error", reject);
+  });
+}
+async function getClientFactory(config) {
+  if (config.createClient) {
+    return config.createClient;
+  }
+  const {
+    createReactionaryFeedClientFactoryFromEnv
+  } = await import("./lib/env-client-builder.js");
+  return createReactionaryFeedClientFactoryFromEnv();
+}
+function createCliRequestContext() {
+  return {
+    languageContext: {
+      locale: "da-DK",
+      currencyCode: "DKK"
+    },
+    storeIdentifier: {
+      key: "the-good-store"
+    },
+    taxJurisdiction: {
+      countryCode: "DK",
+      stateCode: "",
+      countyCode: "",
+      cityCode: ""
+    },
+    session: {
+      identityContext: {
+        identity: {
+          type: "Anonymous"
+        },
+        lastUpdated: /* @__PURE__ */ new Date()
+      },
+      marketingContext: {
+        identifier: { key: "" },
+        segments: [],
+        blurb: ""
+      }
+    },
+    correlationId: "",
+    isBot: false,
+    clientIp: "",
+    userAgent: "",
+    referrer: ""
+  };
+}
+async function loadConfig(configPath) {
+  const module = await import(pathToFileURL(resolve(configPath)).href);
+  if (!module.default) {
+    throw new Error(`Feed config must export a default object: ${configPath}`);
+  }
+  return module.default;
+}
+function createCliProgressReporter(options) {
+  const label = `${options.feedId}/${options.transformerId}`;
+  const throttleMs = options.stream.isTTY ? 100 : 5e3;
+  let lastRenderedAt = 0;
+  let lastLength = 0;
+  return (progress) => {
+    const now = Date.now();
+    const completed = progress.phase === "completed";
+    if (!completed && now - lastRenderedAt < throttleMs) {
+      return;
+    }
+    const line = formatProgressLine(label, progress);
+    if (options.stream.isTTY) {
+      options.stream.write(`\r${line.padEnd(lastLength)}`);
+      lastLength = line.length;
+      if (completed) {
+        options.stream.write("\n");
+      }
+    } else {
+      options.stream.write(`${line}
+`);
+    }
+    lastRenderedAt = now;
+  };
+}
+function formatProgressLine(label, progress) {
+  const total = progress.totalProducts;
+  const processed = progress.processedProducts;
+  const percent = total === void 0 ? void 0 : total === 0 ? 1 : Math.min(processed / total, 1);
+  const count = total === void 0 ? `${processed}/?` : `${processed}/${total}`;
+  const page = progress.pageNumber && progress.totalPages ? ` page ${progress.pageNumber}/${progress.totalPages}` : "";
+  const status = progress.phase === "completed" ? "done" : progress.phase;
+  const eta = progress.phase === "completed" ? "0s" : formatEta(progress, percent);
+  return [
+    `${label}`,
+    formatProgressBar(percent),
+    count,
+    percent === void 0 ? "" : `${Math.round(percent * 100)}%`,
+    status,
+    page.trim(),
+    `elapsed ${formatDuration(progress.elapsedMs)}`,
+    `eta ${eta}`
+  ].filter(Boolean).join(" ");
+}
+function formatProgressBar(percent) {
+  const width = 24;
+  if (percent === void 0) {
+    return `[${"?".repeat(width)}]`;
+  }
+  const completed = Math.round(percent * width);
+  return `[${"#".repeat(completed)}${"-".repeat(width - completed)}]`;
+}
+function formatEta(progress, percent) {
+  if (percent === void 0 || progress.totalProducts === void 0 || progress.processedProducts === 0) {
+    return "estimating";
+  }
+  const remainingProducts = Math.max(
+    progress.totalProducts - progress.processedProducts,
+    0
+  );
+  const msPerProduct = progress.elapsedMs / progress.processedProducts;
+  return formatDuration(msPerProduct * remainingProducts);
+}
+function formatDuration(milliseconds) {
+  const totalSeconds = Math.max(0, Math.round(milliseconds / 1e3));
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes === 0) {
+    return `${seconds}s`;
+  }
+  return `${minutes}m ${seconds}s`;
+}
+function parseProductConcurrency(value, fallback) {
+  if (!value) {
+    return fallback;
+  }
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 1) {
+    throw new Error(`--product-concurrency must be a positive number, got: ${value}`);
+  }
+  return Math.floor(parsed);
+}
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

package/documentation/basic-product-feeds.config.mjs

@@ -0,0 +1,32 @@
+/**
+ * Minimal example config for the `reactionary-feeds` CLI.
+ *
+ * Copy this file, configure provider environment variables in `.env`, and run:
+ *
+ *   reactionary-feeds list-feeds --config ./feeds.config.mjs
+ *   reactionary-feeds generate --config ./feeds.config.mjs --feed default --output google-merchant-feed=./google-merchant.xml
+ */
+
+export default {
+  defaultFulfillmentCenterKeys: ['default'],
+  productConcurrency: 10,
+  feeds: {
+    default: {
+      languageContext: {
+        locale: 'en-GB',
+        currencyCode: 'EUR',
+      },
+      search: {
+        term: '',
+        facets: [],
+        filters: [],
+        paginationOptions: {
+          pageNumber: 1,
+          pageSize: 50,
+        },
+      },
+      productUrlBase: 'https://shop.example/{lang}/products/{slug}',
+      sellerName: 'Example Shop',
+    },
+  },
+};

package/documentation/multilingual-feeds.config.mjs

@@ -0,0 +1,59 @@
+/**
+ * Multilingual example config for da, nb, sv, fi, and en.
+ *
+ * Configure provider environment variables in `.env`, then run:
+ *
+ *   reactionary-feeds list-feeds --config ./multilingual-feeds.config.mjs
+ *   reactionary-feeds generate --config ./multilingual-feeds.config.mjs --feed da --output google-merchant-feed=./google-merchant-da.xml
+ *   reactionary-feeds generate --config ./multilingual-feeds.config.mjs --feed fi --output acp-product-feed=./products-fi.jsonl --output sitemap-feed=./products-fi.xml
+ */
+
+const languageFeeds = {
+  da: {
+    locale: 'da-DK',
+    currencyCode: 'DKK',
+  },
+  nb: {
+    locale: 'nb-NO',
+    currencyCode: 'NOK',
+  },
+  sv: {
+    locale: 'sv-SE',
+    currencyCode: 'SEK',
+  },
+  fi: {
+    locale: 'fi-FI',
+    currencyCode: 'EUR',
+  },
+  en: {
+    locale: 'en-GB',
+    currencyCode: 'GBP',
+  },
+};
+
+export default {
+  defaultFulfillmentCenterKeys: ['OnlineFfmChannel'],
+  productConcurrency: 10,
+  feeds: Object.fromEntries(
+    Object.entries(languageFeeds).map(([language, definition]) => [
+      language,
+      {
+        languageContext: {
+          locale: definition.locale,
+          currencyCode: definition.currencyCode,
+        },
+        search: {
+          term: '',
+          facets: [],
+          filters: [],
+          paginationOptions: {
+            pageNumber: 1,
+            pageSize: 50,
+          },
+        },
+        productUrlBase: `https://shop.example/${language}/products/{slug}`,
+        sellerName: 'Example Shop',
+      },
+    ]),
+  ),
+};

package/documentation/nextjs-route-example.mjs

@@ -0,0 +1,31 @@
+/**
+ * Example Next.js route module shape for mounting ReactionaryFeedServer.
+ *
+ * Copy the relevant parts into `app/feeds/[...path]/route.ts` or a similar
+ * catch-all route. Replace `createClient` and `feedsConfig` with your actual
+ * project setup.
+ */
+
+import {
+  ReactionaryFeedServer,
+  createReactionaryFeedClientFactoryFromEnv,
+} from '@reactionary/feeds';
+import feedsConfig from './feeds.config.mjs';
+
+const server = new ReactionaryFeedServer(
+  feedsConfig.createClient ?? createReactionaryFeedClientFactoryFromEnv(),
+  {
+    feeds: feedsConfig.feeds,
+    sitemaps: feedsConfig.sitemaps,
+    defaultFulfillmentCenterKeys: feedsConfig.defaultFulfillmentCenterKeys,
+    productConcurrency: feedsConfig.productConcurrency,
+  },
+);
+
+export function GET(request) {
+  return server.fetch(request);
+}
+
+export function HEAD(request) {
+  return server.fetch(request);
+}

package/documentation/sitemaps.config.mjs

@@ -0,0 +1,77 @@
+/**
+ * Example config for generating product feed outputs and serving sitemap sources.
+ *
+ * Product-feed CLI outputs:
+ *
+ *   reactionary-feeds generate --config ./feeds.config.mjs --feed finnish --output sitemap-feed=./products.xml
+ *
+ * Generic sitemap source routes are exposed by ReactionaryFeedServer:
+ *
+ *   GET /sitemaps.xml
+ *   GET /sitemaps/products-fi.xml
+ *   GET /sitemaps/categories-fi.xml
+ *   GET /sitemaps/stores-fi.xml
+ */
+
+const finnishLanguageContext = {
+  locale: 'fi-FI',
+  currencyCode: 'EUR',
+};
+
+export default {
+  defaultFulfillmentCenterKeys: ['default'],
+  productConcurrency: 10,
+  feeds: {
+    finnish: {
+      languageContext: finnishLanguageContext,
+      search: {
+        term: '',
+        facets: [],
+        filters: ['market:fi'],
+        paginationOptions: {
+          pageNumber: 1,
+          pageSize: 50,
+        },
+      },
+      productUrlBase: 'https://shop.example/{lang}/products/{slug}',
+      sellerName: 'Example Shop',
+    },
+  },
+
+  sitemaps: {
+    baseUrl: 'https://shop.example',
+    include: ['products-fi', 'categories-fi', 'stores-fi'],
+    sources: {
+      'products-fi': {
+        type: 'products',
+        feed: 'finnish',
+        changefreq: 'daily',
+        priority: 0.8,
+      },
+
+      'categories-fi': {
+        type: 'categories',
+        languageContext: finnishLanguageContext,
+        urlTemplate: 'https://shop.example/{lang}/categories/{slug}',
+        pageSize: 50,
+        maxDepth: 10,
+        changefreq: 'weekly',
+        priority: 0.7,
+      },
+
+      'stores-fi': {
+        type: 'stores',
+        languageContext: finnishLanguageContext,
+        urlTemplate: 'https://shop.example/{lang}/stores/{slug}',
+        proximity: {
+          longitude: 24.9384,
+          latitude: 60.1699,
+          distance: 250,
+          limit: 100,
+        },
+        changefreq: 'monthly',
+        priority: 0.5,
+      },
+    },
+  },
+};

package/index.js

@@ -0,0 +1,13 @@
+export * from "./lib/default-registry.js";
+export * from "./lib/env-client-builder.js";
+export * from "./lib/feed-generator.js";
+export * from "./lib/feed-server.js";
+export * from "./lib/feed-transformer.js";
+export * from "./lib/feed-types.js";
+export * from "./lib/sitemap-generator.js";
+export * from "./lib/sitemap-xml.js";
+export * from "./lib/load-cwd-env.js";
+export * from "./lib/transformers/acp-product-feed.transformer.js";
+export * from "./lib/transformers/google-merchant-feed.transformer.js";
+export * from "./lib/transformers/pricerunner-feed.transformer.js";
+export * from "./lib/transformers/sitemap-feed.transformer.js";

package/lib/default-registry.js

@@ -0,0 +1,18 @@
+import {
+  ReactionaryFeedTransformerRegistry
+} from "./feed-transformer.js";
+import { acpProductFeedTransformer } from "./transformers/acp-product-feed.transformer.js";
+import { googleMerchantFeedTransformer } from "./transformers/google-merchant-feed.transformer.js";
+import { pricerunnerFeedTransformer } from "./transformers/pricerunner-feed.transformer.js";
+import { sitemapFeedTransformer } from "./transformers/sitemap-feed.transformer.js";
+function createDefaultFeedRegistry() {
+  const registry = new ReactionaryFeedTransformerRegistry();
+  registry.register(acpProductFeedTransformer);
+  registry.register(googleMerchantFeedTransformer);
+  registry.register(sitemapFeedTransformer);
+  registry.register(pricerunnerFeedTransformer);
+  return registry;
+}
+export {
+  createDefaultFeedRegistry
+};