@porulle/plugin-channel-connector 0.48.1 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -1
- package/dist/service.d.ts +89 -1
- package/dist/service.js +503 -37
- package/package.json +4 -4
- package/src/index.ts +9 -0
- package/src/service.ts +581 -34
- package/dist/tsconfig.tsbuildinfo +0 -1
package/dist/service.js
CHANGED
|
@@ -364,6 +364,131 @@ function redactStore(store) {
|
|
|
364
364
|
updatedAt: store.updatedAt,
|
|
365
365
|
};
|
|
366
366
|
}
|
|
367
|
+
/**
|
|
368
|
+
* A hero is streamed inside the page's own invocation, so it is bounded: a 30 MB TIFF a merchant
|
|
369
|
+
* uploaded by mistake must not buffer into a 128 MiB isolate. Anything larger is reported, not
|
|
370
|
+
* stored, and the product still lands — the index reads text first and media later.
|
|
371
|
+
*/
|
|
372
|
+
export const HERO_IMAGE_BYTE_CAP = 1024 * 1024;
|
|
373
|
+
function imageOrder(a, b) {
|
|
374
|
+
return (a.sortOrder ?? 0) - (b.sortOrder ?? 0);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Ruling 2026-09-22: import the hero plus the FIRST photo of each other variant, nothing more.
|
|
378
|
+
* A "blue long dress" query must be able to show the blue variant, and a fourth photo of the red
|
|
379
|
+
* one adds nothing the index can use. Variants are read off the images' own variant references,
|
|
380
|
+
* so a connector that lists images against variants it does not enumerate still gets one each.
|
|
381
|
+
*/
|
|
382
|
+
export function selectImportImages(item) {
|
|
383
|
+
const images = [...(item.images ?? [])].sort(imageOrder);
|
|
384
|
+
const hero = images.find((image) => image.role === "primary") ?? images[0] ?? null;
|
|
385
|
+
if (!hero)
|
|
386
|
+
return { hero: null, perVariant: [] };
|
|
387
|
+
const covered = new Set(hero.variantExternalIds ?? []);
|
|
388
|
+
const usedUrls = new Set([hero.url]);
|
|
389
|
+
const perVariant = [];
|
|
390
|
+
const variantRefs = [...new Set(images.flatMap((image) => image.variantExternalIds ?? []))];
|
|
391
|
+
for (const ref of variantRefs) {
|
|
392
|
+
if (covered.has(ref))
|
|
393
|
+
continue;
|
|
394
|
+
const image = images.find((candidate) => candidate.variantExternalIds?.includes(ref) && !usedUrls.has(candidate.url));
|
|
395
|
+
for (const shown of image?.variantExternalIds ?? [])
|
|
396
|
+
covered.add(shown);
|
|
397
|
+
covered.add(ref);
|
|
398
|
+
if (!image)
|
|
399
|
+
continue;
|
|
400
|
+
usedUrls.add(image.url);
|
|
401
|
+
perVariant.push(image);
|
|
402
|
+
}
|
|
403
|
+
return { hero, perVariant };
|
|
404
|
+
}
|
|
405
|
+
/** Streams a response and refuses mid-stream past `cap`; a lying `content-length` cannot get around it. */
|
|
406
|
+
async function fetchBounded(url, cap) {
|
|
407
|
+
let response;
|
|
408
|
+
try {
|
|
409
|
+
response = await fetch(url);
|
|
410
|
+
}
|
|
411
|
+
catch (error) {
|
|
412
|
+
return { ok: false, reason: "download-failed", detail: error instanceof Error ? error.message : "download failed" };
|
|
413
|
+
}
|
|
414
|
+
if (!response.ok)
|
|
415
|
+
return { ok: false, reason: "download-failed", detail: `download returned ${response.status}` };
|
|
416
|
+
const declared = Number(response.headers.get("content-length"));
|
|
417
|
+
if (Number.isFinite(declared) && declared > cap)
|
|
418
|
+
return { ok: false, reason: "too-large", detail: `content-length ${declared} exceeds ${cap}` };
|
|
419
|
+
const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim() || "image/jpeg";
|
|
420
|
+
const reader = response.body?.getReader();
|
|
421
|
+
if (!reader) {
|
|
422
|
+
const buffer = await response.arrayBuffer();
|
|
423
|
+
if (buffer.byteLength > cap)
|
|
424
|
+
return { ok: false, reason: "too-large", detail: `${buffer.byteLength} bytes exceeds ${cap}` };
|
|
425
|
+
return { ok: true, bytes: new Uint8Array(buffer), contentType };
|
|
426
|
+
}
|
|
427
|
+
const chunks = [];
|
|
428
|
+
let total = 0;
|
|
429
|
+
for (;;) {
|
|
430
|
+
const { done, value } = await reader.read();
|
|
431
|
+
if (done)
|
|
432
|
+
break;
|
|
433
|
+
total += value.byteLength;
|
|
434
|
+
if (total > cap) {
|
|
435
|
+
await reader.cancel();
|
|
436
|
+
return { ok: false, reason: "too-large", detail: `stream exceeded ${cap} bytes` };
|
|
437
|
+
}
|
|
438
|
+
chunks.push(value);
|
|
439
|
+
}
|
|
440
|
+
const bytes = new Uint8Array(total);
|
|
441
|
+
let offset = 0;
|
|
442
|
+
for (const chunk of chunks) {
|
|
443
|
+
bytes.set(chunk, offset);
|
|
444
|
+
offset += chunk.byteLength;
|
|
445
|
+
}
|
|
446
|
+
return { ok: true, bytes, contentType };
|
|
447
|
+
}
|
|
448
|
+
function toImportProduct(item) {
|
|
449
|
+
const attributes = item.attributes?.length
|
|
450
|
+
? item.attributes
|
|
451
|
+
: [{ locale: "en", title: item.title, ...(item.description !== undefined ? { description: item.description } : {}) }];
|
|
452
|
+
return {
|
|
453
|
+
ref: item.externalId,
|
|
454
|
+
slug: item.slug,
|
|
455
|
+
...(item.status !== undefined ? { status: item.status, isVisible: item.status === "active" } : {}),
|
|
456
|
+
metadata: mergeMetadata(undefined, item.metadata ?? {}),
|
|
457
|
+
attributes: attributes.map((attribute) => ({
|
|
458
|
+
locale: attribute.locale,
|
|
459
|
+
title: attribute.title,
|
|
460
|
+
...(attribute.subtitle !== undefined ? { subtitle: attribute.subtitle } : {}),
|
|
461
|
+
...(attribute.description !== undefined ? { description: attribute.description } : {}),
|
|
462
|
+
...(attribute.richDescription !== undefined ? { richDescription: attribute.richDescription } : {}),
|
|
463
|
+
...(attribute.seoTitle !== undefined ? { seoTitle: attribute.seoTitle } : {}),
|
|
464
|
+
...(attribute.seoDescription !== undefined ? { seoDescription: attribute.seoDescription } : {}),
|
|
465
|
+
})),
|
|
466
|
+
...(item.options !== undefined ? {
|
|
467
|
+
options: item.options.map((option) => ({
|
|
468
|
+
name: option.name,
|
|
469
|
+
displayName: option.displayName,
|
|
470
|
+
...(option.sortOrder !== undefined ? { sortOrder: option.sortOrder } : {}),
|
|
471
|
+
values: option.values.map((value) => ({
|
|
472
|
+
value: value.value,
|
|
473
|
+
displayValue: value.displayValue,
|
|
474
|
+
...(value.sortOrder !== undefined ? { sortOrder: value.sortOrder } : {}),
|
|
475
|
+
})),
|
|
476
|
+
})),
|
|
477
|
+
} : {}),
|
|
478
|
+
variants: item.variants.map((variant) => ({
|
|
479
|
+
ref: variant.externalId,
|
|
480
|
+
...(variant.sku !== undefined ? { sku: variant.sku } : {}),
|
|
481
|
+
...(variant.barcode !== undefined ? { barcode: variant.barcode } : {}),
|
|
482
|
+
...(variant.optionValues !== undefined ? { options: variant.optionValues } : {}),
|
|
483
|
+
...(variant.prices !== undefined ? { prices: variant.prices } : {}),
|
|
484
|
+
...(variant.metadata !== undefined ? { metadata: variant.metadata } : {}),
|
|
485
|
+
})),
|
|
486
|
+
...(item.tags !== undefined ? { tags: item.tags } : {}),
|
|
487
|
+
...(item.brand !== undefined ? { brand: item.brand } : {}),
|
|
488
|
+
...(item.categories !== undefined ? { categories: item.categories } : {}),
|
|
489
|
+
ownedFieldPaths: importedFieldPaths(item),
|
|
490
|
+
};
|
|
491
|
+
}
|
|
367
492
|
export class ChannelConnectorService {
|
|
368
493
|
db;
|
|
369
494
|
services;
|
|
@@ -713,7 +838,15 @@ export class ChannelConnectorService {
|
|
|
713
838
|
}
|
|
714
839
|
async upsertOptionAxes(entityId, item, actor) {
|
|
715
840
|
const optionValueIds = new Map();
|
|
716
|
-
|
|
841
|
+
// PROJECTED, not `select()`. Three reasons, and the third is the one that saves statements:
|
|
842
|
+
// the row's other columns are never read; a projection types the locally-constructed row below
|
|
843
|
+
// without a cast; and carrying `displayName`/`sortOrder` is what lets the update be SKIPPED when
|
|
844
|
+
// they already hold. A re-import that changes nothing is the common case for a sync, and it used
|
|
845
|
+
// to issue one UPDATE per option type and one per option value regardless.
|
|
846
|
+
const existingTypes = await this.db
|
|
847
|
+
.select({ id: optionTypes.id, name: optionTypes.name, displayName: optionTypes.displayName, sortOrder: optionTypes.sortOrder })
|
|
848
|
+
.from(optionTypes)
|
|
849
|
+
.where(eq(optionTypes.entityId, entityId));
|
|
717
850
|
let changed = false;
|
|
718
851
|
for (const [typeIndex, sourceType] of (item.options ?? []).entries()) {
|
|
719
852
|
let optionType = existingTypes.find((row) => row.name === sourceType.name);
|
|
@@ -721,18 +854,37 @@ export class ChannelConnectorService {
|
|
|
721
854
|
const created = await this.catalog.createOptionType({ entityId, name: sourceType.name, values: [] }, actor);
|
|
722
855
|
if (!created.ok)
|
|
723
856
|
return PluginErr(created.error.message);
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
857
|
+
// The created row is CONSTRUCTED rather than read back: `createOptionType` returns the id and
|
|
858
|
+
// the name is what we just sent.
|
|
859
|
+
//
|
|
860
|
+
// The two nulls are DELIBERATE PLACEHOLDERS, not a claim about the database. `createOptionType`
|
|
861
|
+
// actually persists `displayName: input.name, sortOrder: 0` (core entity-service.ts:595) and
|
|
862
|
+
// `display_name` is NOT NULL (core catalog/schema.ts:307) — so a fresh row never holds null.
|
|
863
|
+
// Constructing nulls here makes the comparison below unequal on any input, which is what forces
|
|
864
|
+
// the update that writes the caller's real `displayName`/`sortOrder` over those defaults.
|
|
865
|
+
optionType = { id: created.value.id, name: sourceType.name, displayName: null, sortOrder: null };
|
|
728
866
|
existingTypes.push(optionType);
|
|
729
867
|
changed = true;
|
|
730
868
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
869
|
+
const desiredTypeSort = sourceType.sortOrder ?? typeIndex;
|
|
870
|
+
// `?? null` NORMALISES, and it is load-bearing rather than tidy. The stored value is `null` or a
|
|
871
|
+
// string; a connector that omits `displayName` sends `undefined`, and `null !== undefined` is
|
|
872
|
+
// true — so without this every such connector issued one UPDATE per option type on every sync
|
|
873
|
+
// and the conditional bought it nothing. Latent on today's corpus, whose connector always sends
|
|
874
|
+
// both fields.
|
|
875
|
+
const desiredTypeDisplay = sourceType.displayName ?? null;
|
|
876
|
+
if (optionType.displayName !== desiredTypeDisplay || optionType.sortOrder !== desiredTypeSort) {
|
|
877
|
+
await this.db.update(optionTypes).set({
|
|
878
|
+
displayName: sourceType.displayName,
|
|
879
|
+
sortOrder: desiredTypeSort,
|
|
880
|
+
}).where(eq(optionTypes.id, optionType.id));
|
|
881
|
+
optionType.displayName = desiredTypeDisplay;
|
|
882
|
+
optionType.sortOrder = desiredTypeSort;
|
|
883
|
+
}
|
|
884
|
+
const existingValues = await this.db
|
|
885
|
+
.select({ id: optionValues.id, value: optionValues.value, displayValue: optionValues.displayValue, sortOrder: optionValues.sortOrder })
|
|
886
|
+
.from(optionValues)
|
|
887
|
+
.where(eq(optionValues.optionTypeId, optionType.id));
|
|
736
888
|
const valueIds = new Map();
|
|
737
889
|
for (const [valueIndex, sourceValue] of sourceType.values.entries()) {
|
|
738
890
|
let optionValue = existingValues.find((row) => row.value === sourceValue.value);
|
|
@@ -740,17 +892,22 @@ export class ChannelConnectorService {
|
|
|
740
892
|
const created = await this.catalog.createOptionValue({ optionTypeId: optionType.id, value: sourceValue.value }, actor);
|
|
741
893
|
if (!created.ok)
|
|
742
894
|
return PluginErr(created.error.message);
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
optionValue = createdValue;
|
|
895
|
+
// Same placeholder reasoning as the option type above: core persists `displayValue: input.value,
|
|
896
|
+
// sortOrder: 0` (entity-service.ts:615), and the nulls force the update that overwrites them.
|
|
897
|
+
optionValue = { id: created.value.id, value: sourceValue.value, displayValue: null, sortOrder: null };
|
|
747
898
|
existingValues.push(optionValue);
|
|
748
899
|
changed = true;
|
|
749
900
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
901
|
+
const desiredValueSort = sourceValue.sortOrder ?? valueIndex;
|
|
902
|
+
const desiredValueDisplay = sourceValue.displayValue ?? null;
|
|
903
|
+
if (optionValue.displayValue !== desiredValueDisplay || optionValue.sortOrder !== desiredValueSort) {
|
|
904
|
+
await this.db.update(optionValues).set({
|
|
905
|
+
displayValue: sourceValue.displayValue,
|
|
906
|
+
sortOrder: desiredValueSort,
|
|
907
|
+
}).where(eq(optionValues.id, optionValue.id));
|
|
908
|
+
optionValue.displayValue = desiredValueDisplay;
|
|
909
|
+
optionValue.sortOrder = desiredValueSort;
|
|
910
|
+
}
|
|
754
911
|
valueIds.set(sourceValue.value, optionValue.id);
|
|
755
912
|
}
|
|
756
913
|
optionValueIds.set(sourceType.name, valueIds);
|
|
@@ -762,10 +919,52 @@ export class ChannelConnectorService {
|
|
|
762
919
|
let repaired = 0;
|
|
763
920
|
let changed = false;
|
|
764
921
|
const mappings = await this.db.select().from(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "variant"), eq(channelEntityMap.entityId, entityId)));
|
|
922
|
+
// ONE read of the existing option-value rows for every already-mapped variant, instead of one
|
|
923
|
+
// per variant inside the loop. Measured on the deployed Worker at 17.0 calls per product, which
|
|
924
|
+
// is one per offer on a catalogue averaging 13 offers per product. A variant CREATED below is
|
|
925
|
+
// absent from this map and correctly reads as empty: its rows are written in this same pass.
|
|
926
|
+
const mappedVariantIds = mappings
|
|
927
|
+
.map((row) => row.variantId)
|
|
928
|
+
.filter((variantId) => variantId !== null);
|
|
929
|
+
const existingOptionValues = new Map();
|
|
930
|
+
if (mappedVariantIds.length > 0) {
|
|
931
|
+
const rows = await this.db
|
|
932
|
+
.select({ variantId: variantOptionValues.variantId, optionValueId: variantOptionValues.optionValueId })
|
|
933
|
+
.from(variantOptionValues)
|
|
934
|
+
.where(inArray(variantOptionValues.variantId, mappedVariantIds));
|
|
935
|
+
for (const row of rows) {
|
|
936
|
+
const list = existingOptionValues.get(row.variantId) ?? [];
|
|
937
|
+
list.push(row.optionValueId);
|
|
938
|
+
existingOptionValues.set(row.variantId, list);
|
|
939
|
+
}
|
|
940
|
+
}
|
|
765
941
|
for (const sourceVariant of item.variants) {
|
|
766
942
|
const fullSourceVariant = fullItem.variants.find((variant) => variant.externalId === sourceVariant.externalId) ?? sourceVariant;
|
|
767
943
|
let mapping = mappings.find((row) => row.externalId === sourceVariant.externalId);
|
|
768
944
|
let variantId = mapping?.variantId;
|
|
945
|
+
// ADOPT BEFORE CREATE. A variant-kind mapping row whose `variantId` is null has lost its link
|
|
946
|
+
// — the key survived, the target did not. Creating a replacement is what the loop used to do,
|
|
947
|
+
// and it cannot work: the orphaned variant still holds the sku, so `variants_native_org_sku_unique`
|
|
948
|
+
// refuses the insert and the item fails on this and every later sync. The row can never heal.
|
|
949
|
+
//
|
|
950
|
+
// `sku` is the store's own natural key for a variant, so re-resolving by it is what restores
|
|
951
|
+
// the link the null destroyed. Scoped to this entity because a sku is unique per organization
|
|
952
|
+
// and adopting another entity's variant would be worse than failing.
|
|
953
|
+
if (mapping && !variantId && sourceVariant.sku) {
|
|
954
|
+
const [adopted] = await this.db
|
|
955
|
+
.select({ id: variants.id })
|
|
956
|
+
.from(variants)
|
|
957
|
+
.where(and(eq(variants.entityId, entityId), eq(variants.sku, sourceVariant.sku)))
|
|
958
|
+
.limit(1);
|
|
959
|
+
if (adopted) {
|
|
960
|
+
variantId = adopted.id;
|
|
961
|
+
// Write the link back, or the row stays broken and every later sync pays this lookup again.
|
|
962
|
+
await this.db.update(channelEntityMap)
|
|
963
|
+
.set({ variantId })
|
|
964
|
+
.where(eq(channelEntityMap.id, mapping.id));
|
|
965
|
+
mapping.variantId = variantId;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
769
968
|
const createdVariant = !variantId;
|
|
770
969
|
if (!variantId) {
|
|
771
970
|
const options = {};
|
|
@@ -786,18 +985,38 @@ export class ChannelConnectorService {
|
|
|
786
985
|
if (!created.ok)
|
|
787
986
|
return PluginErr(created.error.message);
|
|
788
987
|
variantId = created.value.id;
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
988
|
+
if (mapping) {
|
|
989
|
+
// REPAIR IN PLACE. `variantId` is nullable because this table also holds `kind: "entity"`
|
|
990
|
+
// rows, which legitimately have none — but a VARIANT row with a null `variantId` is a
|
|
991
|
+
// broken invariant, not a state to work around. The row already occupies
|
|
992
|
+
// `channel_entity_map_store_kind_external_unique` on (store, kind, externalId), so the
|
|
993
|
+
// insert below would raise a unique violation and fail the whole item. Updating the row
|
|
994
|
+
// we already hold is the only shape that converges.
|
|
995
|
+
await this.db.update(channelEntityMap).set({
|
|
996
|
+
entityId,
|
|
997
|
+
variantId,
|
|
998
|
+
syncHash: hash(fullSourceVariant),
|
|
999
|
+
}).where(eq(channelEntityMap.id, mapping.id));
|
|
1000
|
+
mapping.entityId = entityId;
|
|
1001
|
+
mapping.variantId = variantId;
|
|
1002
|
+
mapping.syncHash = hash(fullSourceVariant);
|
|
1003
|
+
}
|
|
1004
|
+
else {
|
|
1005
|
+
const [createdMapping] = await this.db.insert(channelEntityMap).values({
|
|
1006
|
+
organizationId: orgId,
|
|
1007
|
+
storeId,
|
|
1008
|
+
kind: "variant",
|
|
1009
|
+
externalId: sourceVariant.externalId,
|
|
1010
|
+
entityId,
|
|
1011
|
+
variantId,
|
|
1012
|
+
syncHash: hash(fullSourceVariant),
|
|
1013
|
+
}).returning();
|
|
1014
|
+
mapping = createdMapping;
|
|
1015
|
+
// Pushed so a payload repeating this externalId resolves the row it just created rather
|
|
1016
|
+
// than creating a second variant for it.
|
|
1017
|
+
if (mapping)
|
|
1018
|
+
mappings.push(mapping);
|
|
1019
|
+
}
|
|
801
1020
|
}
|
|
802
1021
|
if (!variantId) {
|
|
803
1022
|
warnings.push(`Skipped variant "${sourceVariant.externalId}": no local variant mapping exists.`);
|
|
@@ -808,8 +1027,7 @@ export class ChannelConnectorService {
|
|
|
808
1027
|
const desiredOptionValueIds = Object.entries(sourceVariant.optionValues ?? {})
|
|
809
1028
|
.map(([name, value]) => optionValueIds.get(name)?.get(value))
|
|
810
1029
|
.filter((optionValueId) => optionValueId !== undefined);
|
|
811
|
-
const
|
|
812
|
-
const currentIds = currentOptionValues.map((row) => row.optionValueId).sort();
|
|
1030
|
+
const currentIds = [...(existingOptionValues.get(variantId) ?? [])].sort();
|
|
813
1031
|
const desiredIds = [...new Set(desiredOptionValueIds)].sort();
|
|
814
1032
|
if (currentIds.length !== desiredIds.length || currentIds.some((id, index) => id !== desiredIds[index])) {
|
|
815
1033
|
await this.db.delete(variantOptionValues).where(eq(variantOptionValues.variantId, variantId));
|
|
@@ -817,6 +1035,9 @@ export class ChannelConnectorService {
|
|
|
817
1035
|
await this.db.insert(variantOptionValues).values(desiredIds.map((optionValueId) => ({ variantId, optionValueId }))).onConflictDoNothing();
|
|
818
1036
|
repaired += 1;
|
|
819
1037
|
}
|
|
1038
|
+
// The map is the read model for this pass; a payload repeating an externalId must not see
|
|
1039
|
+
// the pre-fetched state after this write.
|
|
1040
|
+
existingOptionValues.set(variantId, [...desiredIds]);
|
|
820
1041
|
changed = true;
|
|
821
1042
|
}
|
|
822
1043
|
if (createdVariant && desiredIds.length > 0) {
|
|
@@ -835,16 +1056,57 @@ export class ChannelConnectorService {
|
|
|
835
1056
|
if (!priced.ok)
|
|
836
1057
|
return PluginErr(priced.error.message);
|
|
837
1058
|
}
|
|
838
|
-
|
|
1059
|
+
// Was UNCONDITIONAL: one UPDATE per variant on every import, including a re-sync where the
|
|
1060
|
+
// variant is byte-identical. `mapping.syncHash` is already in hand from the select above, so
|
|
1061
|
+
// the comparison is free and the write disappears on an unchanged variant — which is the
|
|
1062
|
+
// ordinary case for a store that syncs repeatedly.
|
|
1063
|
+
const nextSyncHash = hash(fullSourceVariant);
|
|
1064
|
+
if (mapping && mapping.syncHash !== nextSyncHash) {
|
|
839
1065
|
await this.db.update(channelEntityMap).set({
|
|
840
|
-
syncHash:
|
|
1066
|
+
syncHash: nextSyncHash,
|
|
841
1067
|
}).where(eq(channelEntityMap.id, mapping.id));
|
|
1068
|
+
mapping.syncHash = nextSyncHash;
|
|
842
1069
|
}
|
|
843
1070
|
}
|
|
844
1071
|
return Ok({ value: variantIds, repaired, changed });
|
|
845
1072
|
}
|
|
1073
|
+
/**
|
|
1074
|
+
* The organization's categories, brands and tags, read ONCE per converge run instead of once per
|
|
1075
|
+
* product.
|
|
1076
|
+
*
|
|
1077
|
+
* `applyTaxonomy` takes an `entityId` and is called unconditionally for every item, and each of
|
|
1078
|
+
* its three lookups was a whole-table read filtered by `organization_id`. Measured on the deployed
|
|
1079
|
+
* Worker: 1.0 call per product per class, three classes, every product — an organization's whole
|
|
1080
|
+
* category list re-read for each of twenty products in a batch that cannot have changed it.
|
|
1081
|
+
*
|
|
1082
|
+
* Rows created DURING the run are appended by the callers below, exactly as they were appended to
|
|
1083
|
+
* the per-product arrays before, so an item that introduces a category is still seen by the next
|
|
1084
|
+
* item. The cache is cleared at the top of `convergeCatalogItems`, so its lifetime is one converge
|
|
1085
|
+
* rather than the lifetime of the service.
|
|
1086
|
+
*
|
|
1087
|
+
* The staleness window widens from one product to one batch: a category created by ANOTHER process
|
|
1088
|
+
* mid-batch is not seen here. That was already true within a product — these lists were always a
|
|
1089
|
+
* snapshot — and the create paths below go through `this.catalog`, which refuses a duplicate slug
|
|
1090
|
+
* rather than writing one. So the failure mode is unchanged in kind and wider in window, which is
|
|
1091
|
+
* the trade this comment exists to state rather than hide.
|
|
1092
|
+
*/
|
|
1093
|
+
taxonomyCache = null;
|
|
1094
|
+
async taxonomyFor(orgId) {
|
|
1095
|
+
// Keyed on orgId as well as presence: one service instance serving two organizations must not
|
|
1096
|
+
// hand the second one the first one's taxonomy.
|
|
1097
|
+
if (this.taxonomyCache?.orgId === orgId)
|
|
1098
|
+
return this.taxonomyCache;
|
|
1099
|
+
const [categoryRows, brandRows, tagRows] = await Promise.all([
|
|
1100
|
+
this.db.select().from(categories).where(eq(categories.organizationId, orgId)),
|
|
1101
|
+
this.db.select().from(brands).where(eq(brands.organizationId, orgId)),
|
|
1102
|
+
this.db.select().from(tags).where(eq(tags.organizationId, orgId)),
|
|
1103
|
+
]);
|
|
1104
|
+
this.taxonomyCache = { orgId, categories: categoryRows, brands: brandRows, tags: tagRows };
|
|
1105
|
+
return this.taxonomyCache;
|
|
1106
|
+
}
|
|
846
1107
|
async applyTaxonomy(orgId, entityId, item, actor, warnings) {
|
|
847
|
-
const
|
|
1108
|
+
const taxonomy = await this.taxonomyFor(orgId);
|
|
1109
|
+
const categoryRows = taxonomy.categories;
|
|
848
1110
|
for (const slug of new Set(item.categories ?? [])) {
|
|
849
1111
|
let category = categoryRows.find((row) => row.slug === slug);
|
|
850
1112
|
if (category?.status === "archived") {
|
|
@@ -865,7 +1127,7 @@ export class ChannelConnectorService {
|
|
|
865
1127
|
if (!linked.ok)
|
|
866
1128
|
return PluginErr(linked.error.message);
|
|
867
1129
|
}
|
|
868
|
-
const brandRows =
|
|
1130
|
+
const brandRows = taxonomy.brands;
|
|
869
1131
|
if (item.brand) {
|
|
870
1132
|
let brand = brandRows.find((row) => row.slug === item.brand);
|
|
871
1133
|
if (!brand) {
|
|
@@ -882,7 +1144,7 @@ export class ChannelConnectorService {
|
|
|
882
1144
|
if (!linked.ok)
|
|
883
1145
|
return PluginErr(linked.error.message);
|
|
884
1146
|
}
|
|
885
|
-
const tagRows =
|
|
1147
|
+
const tagRows = taxonomy.tags;
|
|
886
1148
|
for (const slug of new Set(item.tags ?? [])) {
|
|
887
1149
|
let tag = tagRows.find((row) => row.slug === slug);
|
|
888
1150
|
if (!tag) {
|
|
@@ -1688,6 +1950,206 @@ export class ChannelConnectorService {
|
|
|
1688
1950
|
}
|
|
1689
1951
|
}));
|
|
1690
1952
|
}
|
|
1953
|
+
/**
|
|
1954
|
+
* One page from the connector, nothing written. The host lands the page durably (R2 + its
|
|
1955
|
+
* ledger) and hands it to `convergeCatalogPage` from a queue consumer; the two halves are
|
|
1956
|
+
* separate so a consumer retry never re-fetches the merchant's API.
|
|
1957
|
+
*/
|
|
1958
|
+
async fetchCatalogPage(orgId, storeId, cursor) {
|
|
1959
|
+
const store = await this.getStoreRecord(orgId, storeId);
|
|
1960
|
+
if (!store || store.status !== "connected")
|
|
1961
|
+
return PluginErr("Connected store not found.", "NOT_FOUND");
|
|
1962
|
+
const connector = this.connectors.get(store.provider);
|
|
1963
|
+
if (!connector)
|
|
1964
|
+
return PluginErr(`No connector registered for provider "${store.provider}".`);
|
|
1965
|
+
const page = await connector.importCatalog(store, cursor ?? undefined);
|
|
1966
|
+
if (!page.ok)
|
|
1967
|
+
return PluginErr(page.error.message);
|
|
1968
|
+
return Ok({ items: page.value.items, nextCursor: page.value.nextCursor ?? null });
|
|
1969
|
+
}
|
|
1970
|
+
/**
|
|
1971
|
+
* Converges a page: items this store has never mapped take the import fast path
|
|
1972
|
+
* (`catalog.importProducts`, one transaction, multi-row writes); items already mapped and
|
|
1973
|
+
* unchanged cost nothing; items mapped-but-changed, and orphans (an entity of this store with
|
|
1974
|
+
* the item's slug but no map row), take the editor path, which owns ownership and conflicts.
|
|
1975
|
+
*
|
|
1976
|
+
* Media: only each new item's hero is fetched here, streamed under `HERO_IMAGE_BYTE_CAP`, and
|
|
1977
|
+
* linked at entity level as `primary` plus to the variants it shows. The first photo of every
|
|
1978
|
+
* other variant comes back in `deferredMedia` for the host to land later.
|
|
1979
|
+
*/
|
|
1980
|
+
async convergeCatalogPage(orgId, storeId, items, actor) {
|
|
1981
|
+
const failures = [];
|
|
1982
|
+
const warnings = [];
|
|
1983
|
+
const entityByExternalId = new Map();
|
|
1984
|
+
let unchanged = 0;
|
|
1985
|
+
let updated = 0;
|
|
1986
|
+
const externalIds = [...new Set(items.map((item) => item.externalId))];
|
|
1987
|
+
const mappings = externalIds.length === 0 ? [] : await this.db.select().from(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), inArray(channelEntityMap.externalId, externalIds)));
|
|
1988
|
+
const mappingByExternalId = new Map(mappings.map((row) => [row.externalId, row]));
|
|
1989
|
+
const slugs = [...new Set(items.map((item) => item.slug))];
|
|
1990
|
+
const orphans = slugs.length === 0 ? [] : await this.db.select({ slug: sellableEntities.slug }).from(sellableEntities).where(and(eq(sellableEntities.organizationId, orgId), eq(sellableEntities.sourceStoreId, storeId), inArray(sellableEntities.slug, slugs)));
|
|
1991
|
+
const orphanSlugs = new Set(orphans.map((row) => row.slug));
|
|
1992
|
+
const fresh = [];
|
|
1993
|
+
const editor = [];
|
|
1994
|
+
const seen = new Set();
|
|
1995
|
+
for (const item of items) {
|
|
1996
|
+
if (seen.has(item.externalId)) {
|
|
1997
|
+
failures.push({ externalId: item.externalId, error: "duplicate-in-page: this externalId appears twice in the page." });
|
|
1998
|
+
continue;
|
|
1999
|
+
}
|
|
2000
|
+
seen.add(item.externalId);
|
|
2001
|
+
const mapping = mappingByExternalId.get(item.externalId);
|
|
2002
|
+
if (mapping && mapping.syncHash === hash(item)) {
|
|
2003
|
+
entityByExternalId.set(item.externalId, mapping.entityId);
|
|
2004
|
+
unchanged += 1;
|
|
2005
|
+
}
|
|
2006
|
+
else if (mapping || orphanSlugs.has(item.slug)) {
|
|
2007
|
+
editor.push(item);
|
|
2008
|
+
}
|
|
2009
|
+
else {
|
|
2010
|
+
fresh.push(item);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
if (editor.length > 0) {
|
|
2014
|
+
const result = await this.convergeCatalogItems(orgId, storeId, editor, actor);
|
|
2015
|
+
if (!result.ok)
|
|
2016
|
+
return result;
|
|
2017
|
+
failures.push(...result.value.failures);
|
|
2018
|
+
warnings.push(...result.value.warnings);
|
|
2019
|
+
const failedIds = new Set(result.value.failures.map((failure) => failure.externalId));
|
|
2020
|
+
const survivors = editor.filter((item) => !failedIds.has(item.externalId));
|
|
2021
|
+
survivors.forEach((item, index) => {
|
|
2022
|
+
const entityId = result.value.entityIds[index];
|
|
2023
|
+
if (entityId !== undefined)
|
|
2024
|
+
entityByExternalId.set(item.externalId, entityId);
|
|
2025
|
+
});
|
|
2026
|
+
updated += survivors.length;
|
|
2027
|
+
}
|
|
2028
|
+
const createdItems = [];
|
|
2029
|
+
if (fresh.length > 0) {
|
|
2030
|
+
const report = await this.catalog.importProducts(fresh.map(toImportProduct), { sourceStoreId: storeId, errorPolicy: "reject-failed-rows" }, actor);
|
|
2031
|
+
if (!report.ok)
|
|
2032
|
+
return PluginErr(report.error.message, report.error.code);
|
|
2033
|
+
for (const [index, row] of report.value.rows.entries()) {
|
|
2034
|
+
const item = fresh[index];
|
|
2035
|
+
if (!item)
|
|
2036
|
+
continue;
|
|
2037
|
+
if (row.status === "failed") {
|
|
2038
|
+
failures.push({ externalId: row.ref, error: `${row.code}: ${row.error}` });
|
|
2039
|
+
continue;
|
|
2040
|
+
}
|
|
2041
|
+
warnings.push(...row.warnings);
|
|
2042
|
+
entityByExternalId.set(item.externalId, row.entityId);
|
|
2043
|
+
createdItems.push({ item, entityId: row.entityId, variantIds: row.variantIds });
|
|
2044
|
+
}
|
|
2045
|
+
if (createdItems.length > 0) {
|
|
2046
|
+
const now = new Date();
|
|
2047
|
+
await this.db.insert(channelEntityMap).values(createdItems.flatMap(({ item, entityId, variantIds }) => [
|
|
2048
|
+
{ organizationId: orgId, storeId, kind: "entity", externalId: item.externalId, entityId, syncHash: hash(item), lastSyncedAt: now },
|
|
2049
|
+
...item.variants.flatMap((variant) => {
|
|
2050
|
+
const variantId = variantIds[variant.externalId];
|
|
2051
|
+
return variantId === undefined ? [] : [{ organizationId: orgId, storeId, kind: "variant", externalId: variant.externalId, entityId, variantId, syncHash: hash(variant), lastSyncedAt: now }];
|
|
2052
|
+
}),
|
|
2053
|
+
])).onConflictDoNothing();
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
const media = await this.importHeroes(orgId, createdItems, actor);
|
|
2057
|
+
const entityIds = [];
|
|
2058
|
+
const emitted = new Set();
|
|
2059
|
+
for (const item of items) {
|
|
2060
|
+
const entityId = entityByExternalId.get(item.externalId);
|
|
2061
|
+
if (entityId === undefined || emitted.has(entityId))
|
|
2062
|
+
continue;
|
|
2063
|
+
emitted.add(entityId);
|
|
2064
|
+
entityIds.push(entityId);
|
|
2065
|
+
}
|
|
2066
|
+
return Ok({
|
|
2067
|
+
created: createdItems.length,
|
|
2068
|
+
unchanged,
|
|
2069
|
+
updated,
|
|
2070
|
+
entityIds,
|
|
2071
|
+
failures,
|
|
2072
|
+
heroesImported: media.heroesImported,
|
|
2073
|
+
mediaFailures: media.mediaFailures,
|
|
2074
|
+
deferredMedia: media.deferredMedia,
|
|
2075
|
+
warnings,
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
2078
|
+
async importHeroes(orgId, createdItems, actor) {
|
|
2079
|
+
const mediaFailures = [];
|
|
2080
|
+
const deferredMedia = [];
|
|
2081
|
+
const selections = createdItems.flatMap(({ item, entityId, variantIds }) => {
|
|
2082
|
+
const selection = selectImportImages(item);
|
|
2083
|
+
if (selection.perVariant.length > 0)
|
|
2084
|
+
deferredMedia.push({ externalId: item.externalId, entityId, images: selection.perVariant });
|
|
2085
|
+
return selection.hero ? [{ item, entityId, variantIds, hero: selection.hero }] : [];
|
|
2086
|
+
});
|
|
2087
|
+
if (selections.length === 0)
|
|
2088
|
+
return { heroesImported: 0, mediaFailures, deferredMedia };
|
|
2089
|
+
// One read for every hero the page might already hold (a re-import after the map was lost).
|
|
2090
|
+
const urlHashes = [...new Set(selections.map(({ hero }) => hash(hero.url)))];
|
|
2091
|
+
const existingAssets = await this.db.select({ id: mediaAssets.id, metadata: mediaAssets.metadata }).from(mediaAssets).where(and(eq(mediaAssets.organizationId, orgId), inArray(sql `${mediaAssets.metadata}->>'channelImageUrlHash'`, urlHashes)));
|
|
2092
|
+
const assetByUrlHash = new Map();
|
|
2093
|
+
for (const asset of existingAssets) {
|
|
2094
|
+
const urlHash = asset.metadata?.channelImageUrlHash;
|
|
2095
|
+
if (typeof urlHash === "string")
|
|
2096
|
+
assetByUrlHash.set(urlHash, asset.id);
|
|
2097
|
+
}
|
|
2098
|
+
const outcomes = [];
|
|
2099
|
+
const resolveHero = async ({ item, entityId, variantIds, hero }) => {
|
|
2100
|
+
const urlHash = hash(hero.url);
|
|
2101
|
+
const existing = assetByUrlHash.get(urlHash);
|
|
2102
|
+
if (existing !== undefined) {
|
|
2103
|
+
outcomes.push({ entityId, mediaAssetId: existing, hero, variantIds, imported: false });
|
|
2104
|
+
return;
|
|
2105
|
+
}
|
|
2106
|
+
const fetched = await fetchBounded(hero.url, HERO_IMAGE_BYTE_CAP);
|
|
2107
|
+
const failure = (reason, detail) => {
|
|
2108
|
+
mediaFailures.push({ externalId: item.externalId, ...(hero.externalId !== undefined ? { imageExternalId: hero.externalId } : {}), url: hero.url, reason, detail });
|
|
2109
|
+
};
|
|
2110
|
+
if (!fetched.ok) {
|
|
2111
|
+
failure(fetched.reason, fetched.detail);
|
|
2112
|
+
return;
|
|
2113
|
+
}
|
|
2114
|
+
const extension = fetched.contentType.split("/", 2)[1] ?? "jpg";
|
|
2115
|
+
const uploaded = await this.media.upload({
|
|
2116
|
+
filename: `${hero.externalId ?? urlHash}.${extension}`,
|
|
2117
|
+
contentType: fetched.contentType,
|
|
2118
|
+
data: fetched.bytes.buffer,
|
|
2119
|
+
...(hero.alt !== undefined ? { alt: hero.alt } : {}),
|
|
2120
|
+
metadata: { channelImageUrlHash: urlHash, ...(hero.externalId !== undefined ? { channelImageExternalId: hero.externalId } : {}) },
|
|
2121
|
+
origin: "imported",
|
|
2122
|
+
}, actor);
|
|
2123
|
+
if (!uploaded.ok) {
|
|
2124
|
+
failure(uploaded.error.code === "VALIDATION_FAILED" ? "unsupported" : "storage", uploaded.error.message);
|
|
2125
|
+
return;
|
|
2126
|
+
}
|
|
2127
|
+
assetByUrlHash.set(urlHash, uploaded.value.id);
|
|
2128
|
+
outcomes.push({ entityId, mediaAssetId: uploaded.value.id, hero, variantIds, imported: true });
|
|
2129
|
+
};
|
|
2130
|
+
// Six outbound connections per Worker invocation, two per image (download + storage put).
|
|
2131
|
+
const MAX_IN_FLIGHT = 3;
|
|
2132
|
+
let next = 0;
|
|
2133
|
+
await Promise.all(Array.from({ length: Math.min(MAX_IN_FLIGHT, selections.length) }, async () => {
|
|
2134
|
+
for (;;) {
|
|
2135
|
+
const selection = selections[next];
|
|
2136
|
+
next += 1;
|
|
2137
|
+
if (selection === undefined)
|
|
2138
|
+
return;
|
|
2139
|
+
await resolveHero(selection);
|
|
2140
|
+
}
|
|
2141
|
+
}));
|
|
2142
|
+
if (outcomes.length > 0) {
|
|
2143
|
+
await this.db.insert(entityMedia).values(outcomes.flatMap(({ entityId, mediaAssetId, hero, variantIds }) => [
|
|
2144
|
+
{ entityId, mediaAssetId, role: "primary", sortOrder: hero.sortOrder ?? 0 },
|
|
2145
|
+
...(hero.variantExternalIds ?? []).flatMap((externalId) => {
|
|
2146
|
+
const variantId = variantIds[externalId];
|
|
2147
|
+
return variantId === undefined ? [] : [{ entityId, variantId, mediaAssetId, role: hero.role, sortOrder: hero.sortOrder ?? 0 }];
|
|
2148
|
+
}),
|
|
2149
|
+
])).onConflictDoNothing();
|
|
2150
|
+
}
|
|
2151
|
+
return { heroesImported: outcomes.filter((outcome) => outcome.imported).length, mediaFailures, deferredMedia };
|
|
2152
|
+
}
|
|
1691
2153
|
async importCatalog(orgId, storeId, actor, options) {
|
|
1692
2154
|
const store = await this.getStoreRecord(orgId, storeId);
|
|
1693
2155
|
if (!store || store.status !== "connected") {
|
|
@@ -2089,6 +2551,10 @@ export class ChannelConnectorService {
|
|
|
2089
2551
|
async convergeCatalogItems(orgId, storeId, items, actor, force = false, dryRun = false) {
|
|
2090
2552
|
if (dryRun)
|
|
2091
2553
|
return this.estimateCatalogItems(orgId, storeId, items);
|
|
2554
|
+
// One converge, one taxonomy snapshot. Cleared HERE rather than left to the service's lifetime:
|
|
2555
|
+
// the instance can outlive a batch on a warm isolate, and a taxonomy cached across batches would
|
|
2556
|
+
// go stale in a way nothing reports.
|
|
2557
|
+
this.taxonomyCache = null;
|
|
2092
2558
|
let imported = 0;
|
|
2093
2559
|
let converged = 0;
|
|
2094
2560
|
let entitiesTouched = 0;
|