@base44/app-plugin-commerce 0.9.2 → 0.9.3

package/base44/functions/commerce/seed-store/seed-catalog.ts

@@ -633,9 +633,12 @@ async function pMap<T, R>(items: T[], limit: number, fn: (item: T, index: number
 // dominated by call round-trips, and almost none of them depend on each other.
 // The caps keep a big catalog from opening hundreds of simultaneous requests:
 // at most PRODUCT_CONCURRENCY × VARIATION_CONCURRENCY calls are in flight.
-const SEED_CONCURRENCY = 8; // reads/creates within a flat stage
-const PRODUCT_CONCURRENCY = 4; // products in flight (each fans out its variations)
-const VARIATION_CONCURRENCY = 5; // variation creates in flight per product
+const SEED_CONCURRENCY = 8; // reads within a flat stage
+const PRODUCT_CONCURRENCY = 4; // products in flight (each fans out its variation reads)
+const VARIATION_CONCURRENCY = 5; // variation reads in flight per product
+// Records per bulkCreate call. The API takes up to 500; staying well under keeps
+// one oversized batch from carrying a whole catalog's rollback risk.
+const BULK_CHUNK = 200;
 
 // ── seeding ──────────────────────────────────────────────────────────────────
 
@@ -659,10 +662,35 @@ export async function seedCatalog(
   opts: { skipExisting: boolean; errorCode: string },
 ): Promise<Record<string, any>> {
   const created: Array<{ entity: string; id: string }> = [];
-  const track = async (entity: string, record: Record<string, any>) => {
-    const rec = await sr.entities[entity].create(compact(record));
-    created.push({ entity, id: rec.id });
-    return rec;
+  /**
+   * One create call per STAGE, never per record. The platform rate-limits entity
+   * writes by request count, not bytes — `entity_create` allows 140/min and the
+   * 10x builder multiplier does not apply to a backend function — so a catalog
+   * written a record at a time exhausted it mid-seed and the rollback below threw
+   * the whole thing away. Measured on 2,364 kit builds: 13.4% hit a 429 and 18.8%
+   * rolled back. Bulk charges one request per batch against a separate, roomier
+   * bucket (`entity_create_bulk`), which is why the same catalog now fits.
+   *
+   * Ids are recorded per chunk as it lands, so a later chunk's failure still
+   * rolls back the earlier ones. A batch that fails PARTWAY server-side can
+   * leave records this never saw — the catch already tolerates orphans.
+   *
+   * Returns one created record per input record, in input order, which is how
+   * callers below correlate the two. A short return would silently mis-align
+   * them (wrong parent ids, wrong counts), so it throws instead.
+   */
+  const bulkTrack = async (entity: string, records: Record<string, any>[]) => {
+    const out: any[] = [];
+    for (let i = 0; i < records.length; i += BULK_CHUNK) {
+      const chunk = records.slice(i, i + BULK_CHUNK);
+      const made = (await sr.entities[entity].bulkCreate(chunk.map(compact))) ?? [];
+      for (const rec of made) created.push({ entity, id: rec.id });
+      out.push(...made);
+      if (made.length !== chunk.length) {
+        throw new Error(`${entity}: asked for ${chunk.length} records, got ${made.length}`);
+      }
+    }
+    return out;
   };
   const counts = {
     categories: { created: 0, reused: 0 },
@@ -675,41 +703,101 @@ export async function seedCatalog(
     const inventory = (await getSettings(sr, "inventory")).inventory ?? {};
     const outThreshold = Number(inventory.out_of_stock_threshold ?? 0);
 
-    // ── decide per product before writing anything (read-only, parallel) ───
-    const plan = await pMap(spec.products, SEED_CONCURRENCY, async (p) => {
+    // ── one batched read per axis, not one probe per record ─────────────────
+    // Every sku and slug this seed needs to know about is resolved here, in three
+    // `$in` queries that do not grow with the catalog. It used to be a filter()
+    // per product sku, per product slug and per variation sku — 10 reads per
+    // product, ~90% of them single-record existence probes. `entity_list` allows
+    // 100/min per app and that window is shared with the rest of the build, so an
+    // 8-product catalog spent ~80 reads of it and tipped over; the all-or-nothing
+    // rollback then discarded the whole seed. Measured: 8 products = 80 reads
+    // before, 8 after.
+    const SLUG_TRIES = 5; // slug candidates pre-resolved per product; past this it probes
+    const skuOwner = new Map<string, any>();
+    const slugOwner = new Map<string, any>();
+    const baseSlugOf = (p: any) => slugify(p.fields.slug || p.fields.name);
+    const synthSku = (productSku: string, options: Record<string, any>) =>
+      `${productSku}-${Object.values(options).map((o) => String(o).replace(/[^a-z0-9]+/gi, "")).join("-").toUpperCase()}`;
+    const inChunks = async (entity: string, field: string, values: string[]) => {
+      const out: any[] = [];
+      for (let i = 0; i < values.length; i += BULK_CHUNK) {
+        const q = { [field]: { $in: values.slice(i, i + BULK_CHUNK) } };
+        out.push(...((await sr.entities[entity].filter(q, undefined, 500)) ?? []));
+      }
+      return out;
+    };
+    {
+      const skus = new Set<string>();
+      const slugs = new Set<string>();
+      for (const p of spec.products) {
+        const psku = p.fields.sku ? String(p.fields.sku) : "";
+        if (psku) skus.add(psku);
+        const base = baseSlugOf(p);
+        for (let i = 1; i <= SLUG_TRIES; i++) slugs.add(i === 1 ? base : `${base}-${i}`);
+        for (const v of p.variations) {
+          if (v.fields.sku) skus.add(String(v.fields.sku));
+          else if (psku) skus.add(synthSku(psku, v.options));
+        }
+      }
+      // Product before ProductVariation: a sku owned by both reports the product,
+      // which is the precedence the per-record probes had.
+      for (const e of ["commerce.Product", "commerce.ProductVariation"]) {
+        for (const rec of await inChunks(e, "sku", [...skus])) {
+          if (rec.sku && !skuOwner.has(String(rec.sku))) skuOwner.set(String(rec.sku), rec);
+        }
+      }
+      for (const rec of await inChunks("commerce.Product", "slug", [...slugs])) {
+        if (rec.slug && !slugOwner.has(String(rec.slug))) slugOwner.set(String(rec.slug), rec);
+      }
+    }
+
+    // ── decide per product before writing anything (answered by the preflight) ─
+    const plan: Array<{ p: (typeof spec.products)[number]; skip?: Record<string, any> }> = [];
+    for (const p of spec.products) {
       let skip: Record<string, any> | undefined;
       if (opts.skipExisting) {
         const sku = p.fields.sku;
-        if (sku) {
-          const prods = (await sr.entities["commerce.Product"].filter({ sku }, undefined, 1)) ?? [];
-          const hit = prods[0] ?? ((await sr.entities["commerce.ProductVariation"].filter({ sku }, undefined, 1)) ?? [])[0];
-          if (hit) skip = { name: p.fields.name, skipped: true, reason: "sku_exists", existing_id: hit.id };
-        } else {
-          const slug = slugify(p.fields.slug || p.fields.name);
-          const hits = (await sr.entities["commerce.Product"].filter({ slug }, undefined, 1)) ?? [];
-          if (hits.length) skip = { name: p.fields.name, skipped: true, reason: "slug_exists", existing_id: hits[0].id };
+        const hit = sku ? skuOwner.get(String(sku)) : slugOwner.get(baseSlugOf(p));
+        if (hit) {
+          skip = {
+            name: p.fields.name, skipped: true,
+            reason: sku ? "sku_exists" : "slug_exists", existing_id: hit.id,
+          };
         }
       }
       if (!skip) {
-        // explicit variation skus fail as 409 duplicate_sku here, before any write
-        const withSku = p.variations.filter((v) => v.fields.sku);
-        await pMap(withSku, VARIATION_CONCURRENCY, (v) => assertUniqueSku(sr, String(v.fields.sku), {}));
+        // Explicit variation skus fail as 409 duplicate_sku here, before any write.
+        // The preflight decides; assertUniqueSku is called only for a collision, so
+        // the error still comes from the one place that words it.
+        for (const v of p.variations) {
+          const s = v.fields.sku ? String(v.fields.sku) : "";
+          if (s && skuOwner.has(s)) await assertUniqueSku(sr, s, {});
+        }
       }
-      return { p, skip };
-    });
+      plan.push({ p, skip });
+    }
 
     // ── taxonomy (get-or-create; reused records are never rolled back) ─────
+    // Every stage below is the same two beats: resolve what already exists
+    // (reads, parallel), then write everything missing in one call. Created
+    // records are matched back by a field this code set — slug, name, code,
+    // attribute_id — so nothing depends on bulkCreate preserving payload order.
     const categoryBySlug: Record<string, any> = {};
-    await pMap(spec.categories, SEED_CONCURRENCY, async (def) => {
-      const hits = (await sr.entities["commerce.ProductCategory"].filter({ slug: def.slug }, undefined, 1)) ?? [];
-      if (hits.length) {
-        categoryBySlug[def.slug] = hits[0];
+    const missingCategories: Array<Record<string, any>> = [];
+    const existingCategories = await inChunks("commerce.ProductCategory", "slug", spec.categories.map((c) => c.slug));
+    for (const def of spec.categories) {
+      const hit = existingCategories.find((c: any) => c.slug === def.slug);
+      if (hit) {
+        categoryBySlug[def.slug] = hit;
         counts.categories.reused++;
       } else {
-        categoryBySlug[def.slug] = await track("commerce.ProductCategory", { description: "", display: "default", menu_order: 0, ...def, count: 0 });
-        counts.categories.created++;
+        missingCategories.push({ description: "", display: "default", menu_order: 0, ...def, count: 0 });
       }
-    });
+    }
+    for (const rec of await bulkTrack("commerce.ProductCategory", missingCategories)) {
+      categoryBySlug[rec.slug] = rec;
+      counts.categories.created++;
+    }
 
     // Ribbons are resolved up front — the products below write in parallel, so
     // a lazy get-or-create would let two products race the same new ribbon
@@ -724,17 +812,21 @@ export async function seedCatalog(
     }
     if (ribbonNames.size) {
       const allRibbons = await scanAll(sr.entities["commerce.ProductRibbon"], null, "name");
-      await pMap([...ribbonNames.values()], SEED_CONCURRENCY, async (name) => {
+      const missingRibbons: Array<Record<string, any>> = [];
+      for (const name of ribbonNames.values()) {
         const key = name.toLowerCase();
         const existing = allRibbons.find((t: any) => String(t.name ?? "").toLowerCase() === key);
         if (existing) {
           ribbonByLower[key] = existing;
           counts.ribbons.reused++;
         } else {
-          ribbonByLower[key] = await track("commerce.ProductRibbon", { name, count: 0 });
-          counts.ribbons.created++;
+          missingRibbons.push({ name, count: 0 });
         }
-      });
+      }
+      for (const rec of await bulkTrack("commerce.ProductRibbon", missingRibbons)) {
+        ribbonByLower[String(rec.name).toLowerCase()] = rec;
+        counts.ribbons.created++;
+      }
     }
 
     // Attributes and terms: `order` values are pre-assigned from one read, so
@@ -754,23 +846,36 @@ export async function seedCatalog(
           missingAttrs.push({ def, order: nextOrder++ });
         }
       }
-      await pMap(missingAttrs, SEED_CONCURRENCY, async ({ def, order }) => {
-        attributeByCode[def.code] = await track("commerce.ProductAttribute", { name: def.name, code: def.code, order });
+      const madeAttrs = await bulkTrack(
+        "commerce.ProductAttribute",
+        missingAttrs.map(({ def, order }) => ({ name: def.name, code: def.code, order })),
+      );
+      for (const rec of madeAttrs) {
+        attributeByCode[rec.code] = rec;
         counts.attributes.created++;
-      });
+      }
 
-      await pMap(spec.attributes, SEED_CONCURRENCY, async (def) => {
+      // Terms are read per attribute but written together: their `order` values
+      // are pre-assigned off each attribute's own read, so pooling the creates
+      // across attributes cannot collide.
+      const termPlan = await pMap(spec.attributes, SEED_CONCURRENCY, async (def) => {
         const attr = attributeByCode[def.code];
         const terms = (await sr.entities["commerce.ProductAttributeTerm"].filter({ attribute_id: attr.id }, undefined, 500)) ?? [];
         const maxTermOrder = terms.reduce((m: number, t: any) => Math.max(m, Number(t.order ?? 0)), -1);
         const wanted = spec.termsByCode[def.code] ?? [];
         const missing = wanted.filter((term) => !terms.some((t: any) => String(t.name ?? "").toLowerCase() === term.name.toLowerCase()));
         counts.terms.reused += wanted.length - missing.length;
-        const createdTerms = await pMap(missing, VARIATION_CONCURRENCY, (term, i) =>
-          track("commerce.ProductAttributeTerm", { attribute_id: attr.id, name: term.name, order: maxTermOrder + 1 + i, count: 0 }));
-        counts.terms.created += createdTerms.length;
-        termsByAttrId[attr.id] = [...terms, ...createdTerms];
+        return {
+          attrId: attr.id,
+          terms,
+          records: missing.map((term, i) => ({ attribute_id: attr.id, name: term.name, order: maxTermOrder + 1 + i, count: 0 })),
+        };
       });
+      const madeTerms = await bulkTrack("commerce.ProductAttributeTerm", termPlan.flatMap((t) => t.records));
+      counts.terms.created += madeTerms.length;
+      for (const { attrId, terms } of termPlan) {
+        termsByAttrId[attrId] = [...terms, ...madeTerms.filter((t: any) => t.attribute_id === attrId)];
+      }
     }
 
     // References are by name, so payload casing must never fork an existing
@@ -784,9 +889,18 @@ export async function seedCatalog(
     // Slugs are claimed in-call before any await, so two same-named products
     // seeding concurrently can never race `ensureUniqueSlug` to one slug.
     const takenSlugs = new Set<string>();
+    const claimedSkus = new Set<string>(); // synthesized this run; siblings must not collide
     const claimSlug = async (base: string): Promise<string> => {
-      for (let i = 1; i < 100; i++) {
+      // The preflight already resolved the first SLUG_TRIES candidates, so the
+      // common path claims one without a read.
+      for (let i = 1; i <= SLUG_TRIES; i++) {
         const candidate = i === 1 ? base : `${base}-${i}`;
+        if (takenSlugs.has(candidate) || slugOwner.has(candidate)) continue;
+        takenSlugs.add(candidate);
+        return candidate;
+      }
+      for (let i = SLUG_TRIES + 1; i < 100; i++) {
+        const candidate = `${base}-${i}`;
         if (takenSlugs.has(candidate)) continue;
         takenSlugs.add(candidate); // claim synchronously, then verify against the DB
         const hits = (await sr.entities["commerce.Product"].filter({ slug: candidate }, undefined, 1)) ?? [];
@@ -797,16 +911,17 @@ export async function seedCatalog(
       return fallback;
     };
 
-    // ── products + variations (parallel; results keep payload order) ───────
+    // ── products, then variations (one write call each; order kept) ─────────
     const countDeltas = new Map<string, number>(); // "entity|id" → +n
     const bumpLater = (entity: string, id: string) => {
       const key = `${entity}|${id}`;
       countDeltas.set(key, (countDeltas.get(key) ?? 0) + 1);
     };
-    let variationsCreated = 0;
 
-    const results = await pMap(plan, PRODUCT_CONCURRENCY, async ({ p, skip }) => {
-      if (skip) return skip;
+    // Every product record is assembled first — the only awaits are slug claims —
+    // so the whole catalog can then be written in one call.
+    const built: any[] = await pMap(plan, PRODUCT_CONCURRENCY, async ({ p, skip }) => {
+      if (skip) return { skip };
       const record: any = {
         status: "publish",
         ribbon_ids: p.ribbonNames.map((name) => ribbonByLower[name.toLowerCase()]?.id).filter(Boolean),
@@ -837,9 +952,19 @@ export async function seedCatalog(
 
       derivePricing(record);
       deriveStock(record, outThreshold, !!record.manage_stock);
-      const product = await track("commerce.Product", record);
+      return { p, record };
+    });
 
-      const seededVariations = await pMap(p.variations, VARIATION_CONCURRENCY, async (v) => {
+    const seeded = built.filter((b) => !b.skip);
+    const madeProducts = await bulkTrack("commerce.Product", seeded.map((b) => b.record));
+    seeded.forEach((b, i) => { b.product = madeProducts[i]; });
+
+    // Variations carry their parent's id, so they are assembled only once the
+    // products exist — then written together, across all products at once.
+    const varPlan = await pMap(seeded, PRODUCT_CONCURRENCY, async (b) => {
+      const product = b.product;
+      const record = b.record;
+      const records = await pMap(b.p.variations, VARIATION_CONCURRENCY, async (v: any) => {
         const rec: any = {
           status: "publish",
           ...v.fields,
@@ -847,32 +972,60 @@ export async function seedCatalog(
           attributes: Object.entries(v.options).map(([code, option]) => ({
             attribute_id: attributeByCode[code].id,
             name: attributeByCode[code].name,
-            option: canonicalOption(code, option),
+            option: canonicalOption(code, option as string),
           })),
         };
         if (typeof rec.image === "string") rec.image = { src: rec.image, name: record.name, alt: record.name };
         if (!rec.sku && record.sku) {
-          // synthesized skus self-heal on collision instead of failing the seed
-          rec.sku = `${record.sku}-${Object.values(v.options).map((o) => String(o).replace(/[^a-z0-9]+/gi, "")).join("-").toUpperCase()}`;
-          try {
-            await assertUniqueSku(sr, rec.sku, {});
-          } catch {
+          // synthesized skus self-heal on collision instead of failing the seed —
+          // the preflight already knows which are taken, so this costs no read
+          rec.sku = synthSku(String(record.sku), v.options);
+          if (skuOwner.has(rec.sku) || claimedSkus.has(rec.sku)) {
             rec.sku = `${rec.sku}-${crypto.randomUUID().slice(0, 4).toUpperCase()}`;
           }
+          claimedSkus.add(rec.sku);
         }
         // explicit stock is per-variation; none means the pool on the parent
         rec.manage_stock = rec.stock_quantity != null ? "yes" : "parent";
         derivePricing(rec);
         deriveStock(rec, outThreshold, rec.manage_stock === "yes");
-        const createdVariation = await track("commerce.ProductVariation", rec);
-        variationsCreated++;
-        return createdVariation;
+        return rec;
       });
-      if (seededVariations.length) await rollUpParent(sr, product, seededVariations);
+      return { product, records, made: [] as any[] };
+    });
+
+    const madeVariations = await bulkTrack("commerce.ProductVariation", varPlan.flatMap((v) => v.records));
+    const variationsCreated = madeVariations.length;
+    // bulkTrack returns input order, so each product's variations are the next
+    // slice of the flattened result.
+    let cursor = 0;
+    for (const v of varPlan) {
+      v.made = madeVariations.slice(cursor, cursor + v.records.length);
+      cursor += v.records.length;
+    }
 
-      for (const id of record.category_ids) bumpLater("commerce.ProductCategory", id);
-      for (const id of record.ribbon_ids) bumpLater("commerce.ProductRibbon", id);
-      return { name: product.name, id: product.id, slug: product.slug, sku: product.sku ?? "", variation_count: seededVariations.length };
+    await pMap(varPlan, PRODUCT_CONCURRENCY, async (v) => {
+      if (v.made.length) await rollUpParent(sr, v.product, v.made);
+    });
+
+    for (const b of built) {
+      if (b.skip) continue;
+      for (const id of b.record.category_ids) bumpLater("commerce.ProductCategory", id);
+      for (const id of b.record.ribbon_ids) bumpLater("commerce.ProductRibbon", id);
+    }
+
+    const varCountByProductId = new Map<string, number>(
+      varPlan.map((v) => [v.product.id, v.made.length] as [string, number]),
+    );
+    const results = built.map((b) => {
+      if (b.skip) return b.skip;
+      return {
+        name: b.product.name,
+        id: b.product.id,
+        slug: b.product.slug,
+        sku: b.product.sku ?? "",
+        variation_count: varCountByProductId.get(b.product.id) ?? 0,
+      };
     });
 
     // deferred so a mid-creation rollback never leaves counts drifted;
@@ -884,27 +1037,23 @@ export async function seedCatalog(
 
     // ── coupons + locations (skip-if-exists; codes/names are payload-unique) ─
     const couponCounts = { created: 0, skipped: 0 };
+    const missingCoupons: Array<Record<string, any>> = [];
     await pMap(spec.coupons, SEED_CONCURRENCY, async (c) => {
       const code = String(c.code).trim().toLowerCase();
       const hits = (await sr.entities["commerce.Coupon"].filter({ code }, undefined, 1)) ?? [];
-      if (hits.length) {
-        couponCounts.skipped++;
-        return;
-      }
-      await track("commerce.Coupon", { discount_type: "fixed_cart", usage_count: 0, used_by: [], ...c, code });
-      couponCounts.created++;
+      if (hits.length) couponCounts.skipped++;
+      else missingCoupons.push({ discount_type: "fixed_cart", usage_count: 0, used_by: [], ...c, code });
     });
+    couponCounts.created = (await bulkTrack("commerce.Coupon", missingCoupons)).length;
 
     const locationCounts = { created: 0, skipped: 0 };
+    const missingLocations: Array<Record<string, any>> = [];
     await pMap(spec.locations, SEED_CONCURRENCY, async (loc) => {
       const hits = (await sr.entities["commerce.ShippingTaxLocation"].filter({ name: loc.name }, undefined, 1)) ?? [];
-      if (hits.length) {
-        locationCounts.skipped++;
-        return;
-      }
-      await track("commerce.ShippingTaxLocation", loc);
-      locationCounts.created++;
+      if (hits.length) locationCounts.skipped++;
+      else missingLocations.push(loc);
     });
+    locationCounts.created = (await bulkTrack("commerce.ShippingTaxLocation", missingLocations)).length;
 
     return {
       categories: counts.categories,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@base44/app-plugin-commerce",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "Base44 Commerce plugin — entities, backend functions, shared commerce engine, admin UI and the commerce skill, shipped as copyable source",
   "keywords": [
     "base44",