@porulle/plugin-channel-connector 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -4,6 +4,8 @@ export type { MockChannelConnectorOptions } from "./mock-connector.js";
4
4
  export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
5
5
  export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
6
6
  export type { CatalogFieldMapping, CatalogFieldMappingInput, CatalogFieldMappingRow, CatalogFieldTarget, } from "./catalog-field-mapping.js";
7
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
8
+ export declare const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
7
9
  export { signState, verifyState } from "./oauth-state.js";
8
10
  export type { BackfillCatalogOptions, BackfillCatalogReport, BuildCatalogPushItemsOptions, BuildCatalogPushItemsResult, CatalogPushAssemblyField, CatalogPushAssemblyImage, CatalogPushAssemblyItem, CatalogPushPreviewBefore, CatalogPushPreviewBeforeStatus, CatalogPushPreviewDiff, CatalogPushPreviewItem, CatalogPushPreviewResult, CatalogPushPreviewUnavailable, PushCatalogToStoreResult, CatalogPushJobResult, CatalogFieldConflict, CatalogFieldSkip, CatalogPushFieldSkip, CatalogPushSkipReason, CatalogConflictState, CatalogWriteSettings, ChannelComplianceData, ChannelConnectorPluginOptions, ChannelStockLine, ExportState, PublicConnectedStore, ReconcileReport, } from "./service.js";
9
11
  export type { OAuthStatePayload, OAuthStateResult } from "./oauth-state.js";
package/dist/index.js CHANGED
@@ -10,6 +10,8 @@ import { oauthStateEventId, signState, verifyState } from "./oauth-state.js";
10
10
  export { mockChannelConnector } from "./mock-connector.js";
11
11
  export { ChannelConnectorService, CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS, CATALOG_PUSH_BATCH_SIZES, CATALOG_PUSH_MAX_ATTEMPTS, canCatalogPushTransition, canExportTransition, catalogPushConcurrencyKey, catalogPushRetryDelayMs, isCatalogPushBreakerOpen, } from "./service.js";
12
12
  export { isValidCatalogMappingFieldPath, matchFieldPath, mergeCatalogFieldMapping, normalizeCatalogFieldMapping, compareCatalogFieldMappingSpecificity, providerCatalogFieldMappingDefaults, selectCatalogFieldMapping, validateCatalogMappingRow, } from "./catalog-field-mapping.js";
13
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
14
+ export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
13
15
  export { signState, verifyState } from "./oauth-state.js";
14
16
  function unwrap(result) {
15
17
  if (result.ok)
@@ -88,13 +90,26 @@ export function channelConnectorPlugin(options = {}) {
88
90
  concurrency: { key: (input) => String(input.storeId), supersedes: true },
89
91
  handler: async ({ input, ctx }) => {
90
92
  const service = new ChannelConnectorService(ctx.db, ctx.services, options);
91
- const result = await service.importCatalog(String(input.orgId), String(input.storeId), createSystemActor(String(input.orgId)));
93
+ const orgId = String(input.orgId);
94
+ const storeId = String(input.storeId);
95
+ const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
96
+ maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
97
+ });
92
98
  if (!result.ok)
93
99
  throw new Error(result.error);
100
+ if (!result.value.exhausted) {
101
+ const jobs = ctx.services.jobs;
102
+ await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
103
+ organizationId: orgId,
104
+ concurrencyKey: storeId,
105
+ supersedes: true,
106
+ });
107
+ }
94
108
  return {
95
109
  output: {
96
110
  imported: result.value.imported,
97
111
  cursor: result.value.cursor,
112
+ exhausted: result.value.exhausted,
98
113
  ...(result.value.warnings ? { warnings: result.value.warnings } : {}),
99
114
  },
100
115
  };
package/dist/schema.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { boolean, index, integer, jsonb, pgTable, text, timestamp, uniqueIndex, uuid, sql, } from "@porulle/core/drizzle";
2
+ import { sellableEntities } from "@porulle/core/schema";
2
3
  export const connectedStores = pgTable("connected_stores", {
3
4
  id: uuid("id").defaultRandom().primaryKey(),
4
5
  organizationId: text("organization_id").notNull(),
@@ -29,7 +30,7 @@ export const channelEntityMap = pgTable("channel_entity_map", {
29
30
  storeId: uuid("store_id").references(() => connectedStores.id, { onDelete: "cascade" }).notNull(),
30
31
  kind: text("kind", { enum: ["entity", "variant"] }).notNull(),
31
32
  externalId: text("external_id").notNull(),
32
- entityId: uuid("entity_id").notNull(),
33
+ entityId: uuid("entity_id").references(() => sellableEntities.id, { onDelete: "cascade" }).notNull(),
33
34
  variantId: uuid("variant_id"),
34
35
  lastSyncedAt: timestamp("last_synced_at", { withTimezone: true }).defaultNow().notNull(),
35
36
  syncHash: text("sync_hash").notNull(),
package/dist/service.d.ts CHANGED
@@ -216,7 +216,17 @@ export declare class ChannelConnectorService {
216
216
  getStore(orgId: string, id: string): Promise<PluginResult<PublicConnectedStore>>;
217
217
  listStores(orgId: string): Promise<PluginResult<PublicConnectedStore[]>>;
218
218
  validateLineStock(orgId: string, lines: ChannelStockLine[], timeoutMs?: number): Promise<void>;
219
- importCatalog(orgId: string, storeId: string, actor: Actor): Promise<PluginResult<{
219
+ importCatalog(orgId: string, storeId: string, actor: Actor, options: {
220
+ maxItems: number;
221
+ }): Promise<PluginResult<{
222
+ imported: number;
223
+ cursor: string | null;
224
+ exhausted: boolean;
225
+ skipped?: CatalogFieldSkip[];
226
+ conflicts?: CatalogFieldConflict[];
227
+ warnings?: string[];
228
+ }>>;
229
+ importCatalog(orgId: string, storeId: string, actor: Actor, options?: undefined): Promise<PluginResult<{
220
230
  imported: number;
221
231
  cursor: string | null;
222
232
  skipped?: CatalogFieldSkip[];
package/dist/service.js CHANGED
@@ -48,6 +48,26 @@ export function isCatalogPushBreakerOpen(breakerState) {
48
48
  function catalogPushBatchSize(provider) {
49
49
  return CATALOG_PUSH_BATCH_SIZES[provider] ?? DEFAULT_CATALOG_PUSH_BATCH_SIZE;
50
50
  }
51
+ function parseImportResumePosition(raw) {
52
+ if (!raw)
53
+ return { pageCursor: null, offset: 0 };
54
+ try {
55
+ const parsed = JSON.parse(raw);
56
+ if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
57
+ return {
58
+ pageCursor: parsed.pageCursor ?? null,
59
+ offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
60
+ };
61
+ }
62
+ }
63
+ catch {
64
+ // Legacy bare page cursor.
65
+ }
66
+ return { pageCursor: raw, offset: 0 };
67
+ }
68
+ function encodeImportResumePosition(position) {
69
+ return JSON.stringify(position);
70
+ }
51
71
  const exportTransitions = {
52
72
  pending: ["exported", "abandoned"],
53
73
  exported: ["confirmed", "failed", "abandoned"],
@@ -72,6 +92,8 @@ export function canCatalogPushTransition(from, to) {
72
92
  function hash(value) {
73
93
  return createHash("sha256").update(JSON.stringify(value)).digest("hex");
74
94
  }
95
+ /** Mid-import map rows carry this until convergence finishes; must not equal any real remote hash. */
96
+ const PENDING_ENTITY_MAP_SYNC_HASH = "";
75
97
  export const CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS = 15 * 60 * 1000;
76
98
  function normalizeCanonicalValue(value) {
77
99
  if (typeof value === "string")
@@ -1536,7 +1558,7 @@ export class ChannelConnectorService {
1536
1558
  }
1537
1559
  }));
1538
1560
  }
1539
- async importCatalog(orgId, storeId, actor) {
1561
+ async importCatalog(orgId, storeId, actor, options) {
1540
1562
  const store = await this.getStoreRecord(orgId, storeId);
1541
1563
  if (!store || store.status !== "connected") {
1542
1564
  return PluginErr("Connected store not found.", "NOT_FOUND");
@@ -1544,28 +1566,97 @@ export class ChannelConnectorService {
1544
1566
  const connector = this.connectors.get(store.provider);
1545
1567
  if (!connector)
1546
1568
  return PluginErr(`No connector registered for provider "${store.provider}".`);
1547
- const items = [];
1548
- let cursor = store.catalogCursor ?? undefined;
1549
- do {
1550
- const page = await connector.importCatalog(store, cursor);
1569
+ const maxItems = options?.maxItems;
1570
+ const bounded = maxItems !== undefined;
1571
+ if (!bounded) {
1572
+ const resume = parseImportResumePosition(store.catalogCursor);
1573
+ const items = [];
1574
+ let pageCursor = resume.pageCursor ?? undefined;
1575
+ do {
1576
+ const page = await connector.importCatalog(store, pageCursor);
1577
+ if (!page.ok)
1578
+ return PluginErr(page.error.message);
1579
+ items.push(...page.value.items);
1580
+ pageCursor = page.value.nextCursor ?? undefined;
1581
+ } while (pageCursor);
1582
+ const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
1583
+ if (!result.ok)
1584
+ return result;
1585
+ await this.db
1586
+ .update(connectedStores)
1587
+ .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
1588
+ .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
1589
+ return Ok({
1590
+ imported: result.value.imported,
1591
+ cursor: null,
1592
+ ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
1593
+ ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
1594
+ ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
1595
+ });
1596
+ }
1597
+ let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
1598
+ let remaining = maxItems;
1599
+ let exhausted = false;
1600
+ let totalImported = 0;
1601
+ const skipped = [];
1602
+ const conflicts = [];
1603
+ const warnings = [];
1604
+ while (remaining > 0) {
1605
+ const page = await connector.importCatalog(store, pageCursor ?? undefined);
1551
1606
  if (!page.ok)
1552
1607
  return PluginErr(page.error.message);
1553
- items.push(...page.value.items);
1554
- cursor = page.value.nextCursor ?? undefined;
1555
- } while (cursor);
1556
- const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
1557
- if (!result.ok)
1558
- return result;
1608
+ const pageItems = page.value.items;
1609
+ const slice = pageItems.slice(offset);
1610
+ const batchSize = Math.min(remaining, slice.length);
1611
+ if (batchSize === 0) {
1612
+ if (page.value.nextCursor) {
1613
+ pageCursor = page.value.nextCursor;
1614
+ offset = 0;
1615
+ continue;
1616
+ }
1617
+ exhausted = true;
1618
+ break;
1619
+ }
1620
+ const batch = slice.slice(0, batchSize);
1621
+ const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
1622
+ if (!result.ok)
1623
+ return result;
1624
+ totalImported += result.value.imported;
1625
+ remaining -= result.value.consumed;
1626
+ skipped.push(...result.value.skipped);
1627
+ conflicts.push(...result.value.conflicts);
1628
+ warnings.push(...result.value.warnings);
1629
+ offset += result.value.consumed;
1630
+ if (offset >= pageItems.length) {
1631
+ if (page.value.nextCursor) {
1632
+ pageCursor = page.value.nextCursor;
1633
+ offset = 0;
1634
+ }
1635
+ else {
1636
+ exhausted = true;
1637
+ }
1638
+ }
1639
+ if (remaining === 0)
1640
+ break;
1641
+ if (exhausted)
1642
+ break;
1643
+ }
1644
+ const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
1559
1645
  await this.db
1560
1646
  .update(connectedStores)
1561
- .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
1647
+ .set({
1648
+ catalogCursor,
1649
+ ...(exhausted ? { lastSyncAt: new Date() } : {}),
1650
+ updatedAt: new Date(),
1651
+ })
1562
1652
  .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
1563
1653
  return Ok({
1564
- imported: result.value.imported,
1565
- cursor: null,
1566
- ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
1567
- ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
1568
- ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
1654
+ imported: totalImported,
1655
+ cursor: catalogCursor,
1656
+ exhausted,
1657
+ ...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
1658
+ ...(conflicts.length > 0 ? { conflicts } : {}),
1659
+ ...(warnings.length > 0 ? { warnings } : {}),
1569
1660
  });
1570
1661
  }
1571
1662
  async promoteLegacyAttributes(orgId, storeId, actor, dryRun) {
@@ -1726,12 +1817,14 @@ export class ChannelConnectorService {
1726
1817
  attributesCreated: 0,
1727
1818
  mediaImported: 0,
1728
1819
  variantsGivenOptionValues: 0,
1820
+ consumed: 0,
1729
1821
  skipped: [],
1730
1822
  conflicts: [],
1731
1823
  warnings: [],
1732
1824
  };
1733
1825
  const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
1734
1826
  for (const item of items) {
1827
+ stats.consumed += 1;
1735
1828
  const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), eq(channelEntityMap.externalId, item.externalId)));
1736
1829
  if (!entityMapping) {
1737
1830
  stats.imported += 1;
@@ -1855,44 +1948,75 @@ export class ChannelConnectorService {
1855
1948
  let attributesCreated = 0;
1856
1949
  let mediaImported = 0;
1857
1950
  let variantsGivenOptionValues = 0;
1951
+ let consumed = 0;
1858
1952
  const skipped = [];
1859
1953
  const conflicts = [];
1860
1954
  const warnings = [];
1861
1955
  for (const item of items) {
1956
+ consumed += 1;
1862
1957
  const remoteHash = hash(item);
1863
1958
  const existing = await this.db
1864
1959
  .select()
1865
1960
  .from(channelEntityMap)
1866
1961
  .where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), eq(channelEntityMap.externalId, item.externalId)));
1867
- const entityMapping = existing.find((entry) => entry.kind === "entity");
1962
+ let entityMapping = existing.find((entry) => entry.kind === "entity");
1868
1963
  let entityId;
1869
1964
  let isNew = false;
1965
+ let adoptedOrphan = false;
1870
1966
  let entityTouched = false;
1871
1967
  let existingEntity;
1872
1968
  if (entityMapping) {
1873
1969
  const [entity] = await this.db.select().from(sellableEntities).where(and(eq(sellableEntities.organizationId, orgId), eq(sellableEntities.id, entityMapping.entityId)));
1874
1970
  if (!entity) {
1875
- warnings.push(`Skipped "${item.externalId}": mapped entity ${entityMapping.entityId} no longer exists.`);
1876
- continue;
1971
+ await this.db.delete(channelEntityMap).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.entityId, entityMapping.entityId)));
1972
+ entityMapping = undefined;
1973
+ }
1974
+ else {
1975
+ entityId = entityMapping.entityId;
1976
+ existingEntity = entity;
1877
1977
  }
1878
- entityId = entityMapping.entityId;
1879
- existingEntity = entity;
1880
1978
  }
1881
- else {
1882
- const status = item.status;
1883
- const entity = await this.catalog.create({
1884
- type: "product",
1885
- slug: item.slug,
1886
- sourceStoreId: storeId,
1887
- metadata: mergeMetadata(undefined, item.metadata ?? {}),
1888
- ...(status !== undefined ? { status, isVisible: status === "active" } : {}),
1889
- }, actor);
1890
- if (!entity.ok)
1891
- return PluginErr(entity.error.message);
1892
- entityId = entity.value.id;
1893
- isNew = true;
1894
- imported += 1;
1895
- entityTouched = true;
1979
+ if (entityId === undefined) {
1980
+ const [orphan] = await this.db.select().from(sellableEntities).where(and(eq(sellableEntities.organizationId, orgId), eq(sellableEntities.sourceStoreId, storeId), eq(sellableEntities.slug, item.slug)));
1981
+ if (orphan) {
1982
+ entityId = orphan.id;
1983
+ existingEntity = orphan;
1984
+ adoptedOrphan = true;
1985
+ }
1986
+ else {
1987
+ const status = item.status;
1988
+ try {
1989
+ entityId = await this.transact(async (tx) => {
1990
+ const txContext = createTxContext(tx, { actor });
1991
+ const created = await this.catalog.create({
1992
+ type: "product",
1993
+ slug: item.slug,
1994
+ sourceStoreId: storeId,
1995
+ metadata: mergeMetadata(undefined, item.metadata ?? {}),
1996
+ ...(status !== undefined ? { status, isVisible: status === "active" } : {}),
1997
+ }, actor, txContext);
1998
+ if (!created.ok)
1999
+ throw new Error(created.error.message);
2000
+ await tx.insert(channelEntityMap).values({
2001
+ organizationId: orgId,
2002
+ storeId,
2003
+ kind: "entity",
2004
+ externalId: item.externalId,
2005
+ entityId: created.value.id,
2006
+ syncHash: PENDING_ENTITY_MAP_SYNC_HASH,
2007
+ heldFieldPaths: [],
2008
+ forcedPushFieldPaths: [],
2009
+ });
2010
+ return created.value.id;
2011
+ });
2012
+ }
2013
+ catch (error) {
2014
+ return PluginErr(error instanceof Error ? error.message : "Failed to create catalog entity.");
2015
+ }
2016
+ isNew = true;
2017
+ imported += 1;
2018
+ entityTouched = true;
2019
+ }
1896
2020
  }
1897
2021
  const ownershipBeforeSeed = await this.catalog.resolveFieldOwners(entityId, storeId);
1898
2022
  const seedPaths = importedFieldPaths(item).filter((path) => !ownershipBeforeSeed.has(path));
@@ -1989,6 +2113,14 @@ export class ChannelConnectorService {
1989
2113
  const latestRevisionAt = revisionMarkers.at(-1)?.createdAt;
1990
2114
  const lastSyncedAt = latestRevisionAt ?? entityMapping?.lastSyncedAt ?? new Date();
1991
2115
  if (isNew) {
2116
+ await this.db.update(channelEntityMap).set({
2117
+ syncHash: remoteHash,
2118
+ lastSyncedAt,
2119
+ heldFieldPaths: heldSharedPaths,
2120
+ forcedPushFieldPaths: survivingForcedPaths,
2121
+ }).where(and(eq(channelEntityMap.organizationId, orgId), eq(channelEntityMap.storeId, storeId), eq(channelEntityMap.kind, "entity"), eq(channelEntityMap.externalId, item.externalId), eq(channelEntityMap.entityId, entityId)));
2122
+ }
2123
+ else if (adoptedOrphan) {
1992
2124
  await this.db.insert(channelEntityMap).values({
1993
2125
  organizationId: orgId,
1994
2126
  storeId,
@@ -2018,6 +2150,7 @@ export class ChannelConnectorService {
2018
2150
  attributesCreated,
2019
2151
  mediaImported,
2020
2152
  variantsGivenOptionValues,
2153
+ consumed,
2021
2154
  skipped,
2022
2155
  conflicts,
2023
2156
  warnings,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@porulle/plugin-channel-connector",
3
- "version": "0.27.0",
3
+ "version": "0.29.0",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "exports": {
@@ -19,7 +19,7 @@
19
19
  "dependencies": {
20
20
  "@hono/zod-openapi": "^1.2.2",
21
21
  "hono": "^4.12.5",
22
- "@porulle/core": "0.27.0"
22
+ "@porulle/core": "0.29.0"
23
23
  },
24
24
  "devDependencies": {
25
25
  "@types/node": "^24.5.2",
package/src/index.ts CHANGED
@@ -73,6 +73,9 @@ export type {
73
73
  CatalogFieldMappingRow,
74
74
  CatalogFieldTarget,
75
75
  } from "./catalog-field-mapping.js";
76
+ /** ~320 Neon HTTP subrequests per product against a 10,000 per-invocation cap → hard ceiling near 31; 20 leaves margin for heavier products. */
77
+ export const CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION = 20;
78
+
76
79
  export { signState, verifyState } from "./oauth-state.js";
77
80
  export type {
78
81
  BackfillCatalogOptions,
@@ -193,12 +196,25 @@ export function channelConnectorPlugin(options: ChannelConnectorPluginOptions =
193
196
  concurrency: { key: (input: Record<string, unknown>) => String(input.storeId), supersedes: true },
194
197
  handler: async ({ input, ctx }: { input: Record<string, unknown>; ctx: import("@porulle/core").TaskContext }) => {
195
198
  const service = new ChannelConnectorService(ctx.db, ctx.services, options);
196
- const result = await service.importCatalog(String(input.orgId), String(input.storeId), createSystemActor(String(input.orgId)));
199
+ const orgId = String(input.orgId);
200
+ const storeId = String(input.storeId);
201
+ const result = await service.importCatalog(orgId, storeId, createSystemActor(orgId), {
202
+ maxItems: CHANNEL_IMPORT_MAX_ITEMS_PER_INVOCATION,
203
+ });
197
204
  if (!result.ok) throw new Error(result.error);
205
+ if (!result.value.exhausted) {
206
+ const jobs = ctx.services.jobs as JobsAdapter;
207
+ await jobs.enqueue("channel/import-catalog", { orgId, storeId }, {
208
+ organizationId: orgId,
209
+ concurrencyKey: storeId,
210
+ supersedes: true,
211
+ });
212
+ }
198
213
  return {
199
214
  output: {
200
215
  imported: result.value.imported,
201
216
  cursor: result.value.cursor,
217
+ exhausted: result.value.exhausted,
202
218
  ...(result.value.warnings ? { warnings: result.value.warnings } : {}),
203
219
  },
204
220
  };
package/src/schema.ts CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  sql,
12
12
  } from "@porulle/core/drizzle";
13
13
  import type { ChannelPushCatalogItem, FieldPath } from "@porulle/core";
14
+ import { sellableEntities } from "@porulle/core/schema";
14
15
  import type { CatalogFieldMapping } from "./catalog-field-mapping.js";
15
16
 
16
17
  export const connectedStores = pgTable(
@@ -50,7 +51,7 @@ export const channelEntityMap = pgTable(
50
51
  storeId: uuid("store_id").references(() => connectedStores.id, { onDelete: "cascade" }).notNull(),
51
52
  kind: text("kind", { enum: ["entity", "variant"] }).notNull(),
52
53
  externalId: text("external_id").notNull(),
53
- entityId: uuid("entity_id").notNull(),
54
+ entityId: uuid("entity_id").references(() => sellableEntities.id, { onDelete: "cascade" }).notNull(),
54
55
  variantId: uuid("variant_id"),
55
56
  lastSyncedAt: timestamp("last_synced_at", { withTimezone: true }).defaultNow().notNull(),
56
57
  syncHash: text("sync_hash").notNull(),
package/src/service.ts CHANGED
@@ -241,11 +241,34 @@ interface CatalogConvergenceStats {
241
241
  attributesCreated: number;
242
242
  mediaImported: number;
243
243
  variantsGivenOptionValues: number;
244
+ consumed: number;
244
245
  skipped: CatalogFieldSkip[];
245
246
  conflicts: CatalogFieldConflict[];
246
247
  warnings: string[];
247
248
  }
248
249
 
250
+ type ImportResumePosition = { pageCursor: string | null; offset: number };
251
+
252
+ function parseImportResumePosition(raw: string | null | undefined): ImportResumePosition {
253
+ if (!raw) return { pageCursor: null, offset: 0 };
254
+ try {
255
+ const parsed = JSON.parse(raw) as Partial<ImportResumePosition>;
256
+ if (typeof parsed === "object" && parsed !== null && "offset" in parsed) {
257
+ return {
258
+ pageCursor: parsed.pageCursor ?? null,
259
+ offset: typeof parsed.offset === "number" && parsed.offset >= 0 ? parsed.offset : 0,
260
+ };
261
+ }
262
+ } catch {
263
+ // Legacy bare page cursor.
264
+ }
265
+ return { pageCursor: raw, offset: 0 };
266
+ }
267
+
268
+ function encodeImportResumePosition(position: ImportResumePosition): string {
269
+ return JSON.stringify(position);
270
+ }
271
+
249
272
  export interface BackfillCatalogOptions {
250
273
  dryRun?: boolean;
251
274
  resume?: boolean;
@@ -343,6 +366,7 @@ interface CatalogService {
343
366
  isVisible?: boolean;
344
367
  },
345
368
  actor: Actor,
369
+ ctx?: TxContext,
346
370
  ): Promise<{ ok: true; value: { id: string } } | { ok: false; error: { message: string } }>;
347
371
  createVariant(
348
372
  input: { entityId: string; options: Record<string, string>; sku?: string; barcode?: string },
@@ -481,6 +505,9 @@ function hash(value: unknown): string {
481
505
  return createHash("sha256").update(JSON.stringify(value)).digest("hex");
482
506
  }
483
507
 
508
+ /** Mid-import map rows carry this until convergence finishes; must not equal any real remote hash. */
509
+ const PENDING_ENTITY_MAP_SYNC_HASH = "";
510
+
484
511
  export const CATALOG_OUTBOUND_SUPPRESSION_WINDOW_MS = 15 * 60 * 1000;
485
512
 
486
513
  function normalizeCanonicalValue(value: unknown): unknown {
@@ -2171,7 +2198,40 @@ export class ChannelConnectorService {
2171
2198
  orgId: string,
2172
2199
  storeId: string,
2173
2200
  actor: Actor,
2174
- ): Promise<PluginResult<{ imported: number; cursor: string | null; skipped?: CatalogFieldSkip[]; conflicts?: CatalogFieldConflict[]; warnings?: string[] }>> {
2201
+ options: { maxItems: number },
2202
+ ): Promise<PluginResult<{
2203
+ imported: number;
2204
+ cursor: string | null;
2205
+ exhausted: boolean;
2206
+ skipped?: CatalogFieldSkip[];
2207
+ conflicts?: CatalogFieldConflict[];
2208
+ warnings?: string[];
2209
+ }>>;
2210
+ async importCatalog(
2211
+ orgId: string,
2212
+ storeId: string,
2213
+ actor: Actor,
2214
+ options?: undefined,
2215
+ ): Promise<PluginResult<{
2216
+ imported: number;
2217
+ cursor: string | null;
2218
+ skipped?: CatalogFieldSkip[];
2219
+ conflicts?: CatalogFieldConflict[];
2220
+ warnings?: string[];
2221
+ }>>;
2222
+ async importCatalog(
2223
+ orgId: string,
2224
+ storeId: string,
2225
+ actor: Actor,
2226
+ options?: { maxItems?: number },
2227
+ ): Promise<PluginResult<{
2228
+ imported: number;
2229
+ cursor: string | null;
2230
+ exhausted?: boolean;
2231
+ skipped?: CatalogFieldSkip[];
2232
+ conflicts?: CatalogFieldConflict[];
2233
+ warnings?: string[];
2234
+ }>> {
2175
2235
  const store = await this.getStoreRecord(orgId, storeId);
2176
2236
  if (!store || store.status !== "connected") {
2177
2237
  return PluginErr("Connected store not found.", "NOT_FOUND");
@@ -2179,28 +2239,103 @@ export class ChannelConnectorService {
2179
2239
  const connector = this.connectors.get(store.provider);
2180
2240
  if (!connector) return PluginErr(`No connector registered for provider "${store.provider}".`);
2181
2241
 
2182
- const items: ChannelCatalogItem[] = [];
2183
- let cursor: string | undefined = store.catalogCursor ?? undefined;
2184
- do {
2185
- const page = await connector.importCatalog(store as ChannelStore, cursor);
2242
+ const maxItems = options?.maxItems;
2243
+ const bounded = maxItems !== undefined;
2244
+
2245
+ if (!bounded) {
2246
+ const resume = parseImportResumePosition(store.catalogCursor);
2247
+ const items: ChannelCatalogItem[] = [];
2248
+ let pageCursor: string | undefined = resume.pageCursor ?? undefined;
2249
+ do {
2250
+ const page = await connector.importCatalog(store as ChannelStore, pageCursor);
2251
+ if (!page.ok) return PluginErr(page.error.message);
2252
+ items.push(...page.value.items);
2253
+ pageCursor = page.value.nextCursor ?? undefined;
2254
+ } while (pageCursor);
2255
+
2256
+ const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
2257
+ if (!result.ok) return result;
2258
+
2259
+ await this.db
2260
+ .update(connectedStores)
2261
+ .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
2262
+ .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
2263
+ return Ok({
2264
+ imported: result.value.imported,
2265
+ cursor: null,
2266
+ ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
2267
+ ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
2268
+ ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
2269
+ });
2270
+ }
2271
+
2272
+ let { pageCursor, offset } = parseImportResumePosition(store.catalogCursor);
2273
+ let remaining = maxItems;
2274
+ let exhausted = false;
2275
+ let totalImported = 0;
2276
+ const skipped: CatalogFieldSkip[] = [];
2277
+ const conflicts: CatalogFieldConflict[] = [];
2278
+ const warnings: string[] = [];
2279
+
2280
+ while (remaining > 0) {
2281
+ const page = await connector.importCatalog(store as ChannelStore, pageCursor ?? undefined);
2186
2282
  if (!page.ok) return PluginErr(page.error.message);
2187
- items.push(...page.value.items);
2188
- cursor = page.value.nextCursor ?? undefined;
2189
- } while (cursor);
2190
2283
 
2191
- const result = await this.convergeCatalogItems(orgId, storeId, items, actor);
2192
- if (!result.ok) return result;
2284
+ const pageItems = page.value.items;
2285
+ const slice = pageItems.slice(offset);
2286
+ const batchSize = Math.min(remaining, slice.length);
2287
+
2288
+ if (batchSize === 0) {
2289
+ if (page.value.nextCursor) {
2290
+ pageCursor = page.value.nextCursor;
2291
+ offset = 0;
2292
+ continue;
2293
+ }
2294
+ exhausted = true;
2295
+ break;
2296
+ }
2297
+
2298
+ const batch = slice.slice(0, batchSize);
2299
+ const result = await this.convergeCatalogItems(orgId, storeId, batch, actor);
2300
+ if (!result.ok) return result;
2301
+
2302
+ totalImported += result.value.imported;
2303
+ remaining -= result.value.consumed;
2304
+ skipped.push(...result.value.skipped);
2305
+ conflicts.push(...result.value.conflicts);
2306
+ warnings.push(...result.value.warnings);
2307
+ offset += result.value.consumed;
2308
+
2309
+ if (offset >= pageItems.length) {
2310
+ if (page.value.nextCursor) {
2311
+ pageCursor = page.value.nextCursor;
2312
+ offset = 0;
2313
+ } else {
2314
+ exhausted = true;
2315
+ }
2316
+ }
2193
2317
 
2318
+ if (remaining === 0) break;
2319
+ if (exhausted) break;
2320
+ }
2321
+
2322
+ const catalogCursor = exhausted ? null : encodeImportResumePosition({ pageCursor, offset });
2194
2323
  await this.db
2195
2324
  .update(connectedStores)
2196
- .set({ catalogCursor: null, lastSyncAt: new Date(), updatedAt: new Date() })
2325
+ .set({
2326
+ catalogCursor,
2327
+ ...(exhausted ? { lastSyncAt: new Date() } : {}),
2328
+ updatedAt: new Date(),
2329
+ })
2197
2330
  .where(and(eq(connectedStores.organizationId, orgId), eq(connectedStores.id, storeId)));
2331
+
2198
2332
  return Ok({
2199
- imported: result.value.imported,
2200
- cursor: null,
2201
- ...(result.value.skipped.length > 0 ? { skipped: uniqueSkipped(result.value.skipped) } : {}),
2202
- ...(result.value.conflicts.length > 0 ? { conflicts: result.value.conflicts } : {}),
2203
- ...(result.value.warnings.length > 0 ? { warnings: result.value.warnings } : {}),
2333
+ imported: totalImported,
2334
+ cursor: catalogCursor,
2335
+ exhausted,
2336
+ ...(skipped.length > 0 ? { skipped: uniqueSkipped(skipped) } : {}),
2337
+ ...(conflicts.length > 0 ? { conflicts } : {}),
2338
+ ...(warnings.length > 0 ? { warnings } : {}),
2204
2339
  });
2205
2340
  }
2206
2341
 
@@ -2384,12 +2519,14 @@ export class ChannelConnectorService {
2384
2519
  attributesCreated: 0,
2385
2520
  mediaImported: 0,
2386
2521
  variantsGivenOptionValues: 0,
2522
+ consumed: 0,
2387
2523
  skipped: [],
2388
2524
  conflicts: [],
2389
2525
  warnings: [],
2390
2526
  };
2391
2527
  const assets = await this.db.select().from(mediaAssets).where(eq(mediaAssets.organizationId, orgId));
2392
2528
  for (const item of items) {
2529
+ stats.consumed += 1;
2393
2530
  const [entityMapping] = await this.db.select().from(channelEntityMap).where(and(
2394
2531
  eq(channelEntityMap.organizationId, orgId),
2395
2532
  eq(channelEntityMap.storeId, storeId),
@@ -2529,10 +2666,12 @@ export class ChannelConnectorService {
2529
2666
  let attributesCreated = 0;
2530
2667
  let mediaImported = 0;
2531
2668
  let variantsGivenOptionValues = 0;
2669
+ let consumed = 0;
2532
2670
  const skipped: CatalogFieldSkip[] = [];
2533
2671
  const conflicts: CatalogFieldConflict[] = [];
2534
2672
  const warnings: string[] = [];
2535
2673
  for (const item of items) {
2674
+ consumed += 1;
2536
2675
  const remoteHash = hash(item);
2537
2676
  const existing = await this.db
2538
2677
  .select()
@@ -2543,9 +2682,10 @@ export class ChannelConnectorService {
2543
2682
  eq(channelEntityMap.kind, "entity"),
2544
2683
  eq(channelEntityMap.externalId, item.externalId),
2545
2684
  ));
2546
- const entityMapping = existing.find((entry) => entry.kind === "entity");
2547
- let entityId: string;
2685
+ let entityMapping = existing.find((entry) => entry.kind === "entity");
2686
+ let entityId: string | undefined;
2548
2687
  let isNew = false;
2688
+ let adoptedOrphan = false;
2549
2689
  let entityTouched = false;
2550
2690
  let existingEntity: typeof sellableEntities.$inferSelect | undefined;
2551
2691
  if (entityMapping) {
@@ -2554,25 +2694,59 @@ export class ChannelConnectorService {
2554
2694
  eq(sellableEntities.id, entityMapping.entityId),
2555
2695
  ));
2556
2696
  if (!entity) {
2557
- warnings.push(`Skipped "${item.externalId}": mapped entity ${entityMapping.entityId} no longer exists.`);
2558
- continue;
2697
+ await this.db.delete(channelEntityMap).where(and(
2698
+ eq(channelEntityMap.organizationId, orgId),
2699
+ eq(channelEntityMap.storeId, storeId),
2700
+ eq(channelEntityMap.entityId, entityMapping.entityId),
2701
+ ));
2702
+ entityMapping = undefined;
2703
+ } else {
2704
+ entityId = entityMapping.entityId;
2705
+ existingEntity = entity;
2706
+ }
2707
+ }
2708
+ if (entityId === undefined) {
2709
+ const [orphan] = await this.db.select().from(sellableEntities).where(and(
2710
+ eq(sellableEntities.organizationId, orgId),
2711
+ eq(sellableEntities.sourceStoreId, storeId),
2712
+ eq(sellableEntities.slug, item.slug),
2713
+ ));
2714
+ if (orphan) {
2715
+ entityId = orphan.id;
2716
+ existingEntity = orphan;
2717
+ adoptedOrphan = true;
2718
+ } else {
2719
+ const status = item.status;
2720
+ try {
2721
+ entityId = await this.transact(async (tx) => {
2722
+ const txContext = createTxContext(tx, { actor });
2723
+ const created = await this.catalog.create({
2724
+ type: "product",
2725
+ slug: item.slug,
2726
+ sourceStoreId: storeId,
2727
+ metadata: mergeMetadata(undefined, item.metadata ?? {}),
2728
+ ...(status !== undefined ? { status, isVisible: status === "active" } : {}),
2729
+ }, actor, txContext);
2730
+ if (!created.ok) throw new Error(created.error.message);
2731
+ await tx.insert(channelEntityMap).values({
2732
+ organizationId: orgId,
2733
+ storeId,
2734
+ kind: "entity",
2735
+ externalId: item.externalId,
2736
+ entityId: created.value.id,
2737
+ syncHash: PENDING_ENTITY_MAP_SYNC_HASH,
2738
+ heldFieldPaths: [],
2739
+ forcedPushFieldPaths: [],
2740
+ });
2741
+ return created.value.id;
2742
+ });
2743
+ } catch (error) {
2744
+ return PluginErr(error instanceof Error ? error.message : "Failed to create catalog entity.");
2745
+ }
2746
+ isNew = true;
2747
+ imported += 1;
2748
+ entityTouched = true;
2559
2749
  }
2560
- entityId = entityMapping.entityId;
2561
- existingEntity = entity;
2562
- } else {
2563
- const status = item.status;
2564
- const entity = await this.catalog.create({
2565
- type: "product",
2566
- slug: item.slug,
2567
- sourceStoreId: storeId,
2568
- metadata: mergeMetadata(undefined, item.metadata ?? {}),
2569
- ...(status !== undefined ? { status, isVisible: status === "active" } : {}),
2570
- }, actor);
2571
- if (!entity.ok) return PluginErr(entity.error.message);
2572
- entityId = entity.value.id;
2573
- isNew = true;
2574
- imported += 1;
2575
- entityTouched = true;
2576
2750
  }
2577
2751
 
2578
2752
  const ownershipBeforeSeed = await this.catalog.resolveFieldOwners(entityId, storeId);
@@ -2684,6 +2858,19 @@ export class ChannelConnectorService {
2684
2858
  const lastSyncedAt = latestRevisionAt ?? entityMapping?.lastSyncedAt ?? new Date();
2685
2859
 
2686
2860
  if (isNew) {
2861
+ await this.db.update(channelEntityMap).set({
2862
+ syncHash: remoteHash,
2863
+ lastSyncedAt,
2864
+ heldFieldPaths: heldSharedPaths,
2865
+ forcedPushFieldPaths: survivingForcedPaths,
2866
+ }).where(and(
2867
+ eq(channelEntityMap.organizationId, orgId),
2868
+ eq(channelEntityMap.storeId, storeId),
2869
+ eq(channelEntityMap.kind, "entity"),
2870
+ eq(channelEntityMap.externalId, item.externalId),
2871
+ eq(channelEntityMap.entityId, entityId),
2872
+ ));
2873
+ } else if (adoptedOrphan) {
2687
2874
  await this.db.insert(channelEntityMap).values({
2688
2875
  organizationId: orgId,
2689
2876
  storeId,
@@ -2717,6 +2904,7 @@ export class ChannelConnectorService {
2717
2904
  attributesCreated,
2718
2905
  mediaImported,
2719
2906
  variantsGivenOptionValues,
2907
+ consumed,
2720
2908
  skipped,
2721
2909
  conflicts,
2722
2910
  warnings,